2024-11-17 20:23:42,701 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 20:23:42,716 main DEBUG Took 0.012647 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 20:23:42,716 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 20:23:42,717 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 20:23:42,718 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 20:23:42,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,729 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 20:23:42,746 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,748 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,749 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,751 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,752 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,753 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,755 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,755 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,756 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,757 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,757 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,758 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,758 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,759 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,759 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,760 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,761 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,761 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 20:23:42,762 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,762 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 20:23:42,764 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 20:23:42,765 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 20:23:42,768 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 20:23:42,768 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 20:23:42,770 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 20:23:42,770 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 20:23:42,780 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 20:23:42,784 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 20:23:42,786 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 20:23:42,787 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 20:23:42,787 main DEBUG createAppenders(={Console}) 2024-11-17 20:23:42,788 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 20:23:42,789 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 20:23:42,789 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 20:23:42,790 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 20:23:42,790 main DEBUG OutputStream closed 2024-11-17 20:23:42,791 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 20:23:42,791 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 20:23:42,791 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 20:23:42,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 20:23:42,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 20:23:42,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 20:23:42,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 20:23:42,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 20:23:42,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 20:23:42,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 20:23:42,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 20:23:42,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 20:23:42,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 20:23:42,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 20:23:42,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 20:23:42,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 20:23:42,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 20:23:42,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 20:23:42,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 20:23:42,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 20:23:42,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 20:23:42,910 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 20:23:42,910 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 20:23:42,911 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 20:23:42,912 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T20:23:43,273 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5 2024-11-17 20:23:43,277 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 20:23:43,278 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T20:23:43,289 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-17T20:23:43,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T20:23:43,338 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac, deleteOnExit=true 2024-11-17T20:23:43,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T20:23:43,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/test.cache.data in system properties and HBase conf 2024-11-17T20:23:43,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T20:23:43,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/hadoop.log.dir in system properties and HBase conf 2024-11-17T20:23:43,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T20:23:43,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T20:23:43,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T20:23:43,456 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T20:23:43,541 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T20:23:43,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T20:23:43,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T20:23:43,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T20:23:43,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T20:23:43,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T20:23:43,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T20:23:43,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T20:23:43,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T20:23:43,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T20:23:43,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/nfs.dump.dir in system properties and HBase conf 2024-11-17T20:23:43,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/java.io.tmpdir in system properties and HBase conf 2024-11-17T20:23:43,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T20:23:43,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T20:23:43,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T20:23:44,839 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T20:23:44,917 INFO [Time-limited test {}] log.Log(170): Logging initialized @3200ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T20:23:44,993 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T20:23:45,051 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T20:23:45,073 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T20:23:45,074 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T20:23:45,075 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T20:23:45,090 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T20:23:45,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/hadoop.log.dir/,AVAILABLE} 2024-11-17T20:23:45,094 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T20:23:45,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/java.io.tmpdir/jetty-localhost-41925-hadoop-hdfs-3_4_1-tests_jar-_-any-8233014000177531424/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T20:23:45,305 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41925} 2024-11-17T20:23:45,305 INFO [Time-limited test {}] server.Server(415): Started @3590ms 2024-11-17T20:23:45,829 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T20:23:45,839 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T20:23:45,840 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T20:23:45,840 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T20:23:45,841 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T20:23:45,841 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/hadoop.log.dir/,AVAILABLE} 2024-11-17T20:23:45,842 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T20:23:45,939 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/java.io.tmpdir/jetty-localhost-33403-hadoop-hdfs-3_4_1-tests_jar-_-any-17013022085430864024/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T20:23:45,939 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:33403} 2024-11-17T20:23:45,940 INFO [Time-limited test {}] server.Server(415): Started @4224ms 2024-11-17T20:23:45,986 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T20:23:47,260 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/data/data2/current/BP-1443374901-172.17.0.2-1731875024098/current, will proceed with Du for space computation calculation, 2024-11-17T20:23:47,260 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/data/data1/current/BP-1443374901-172.17.0.2-1731875024098/current, will proceed with Du for space computation calculation, 2024-11-17T20:23:47,285 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T20:23:47,328 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4742724fdd95ce7a with lease ID 0xeda9b1da7cc5716: Processing first storage report for DS-96187b8f-658b-441b-a2a5-378169f6917f from datanode DatanodeRegistration(127.0.0.1:38093, datanodeUuid=1404d3bf-c966-4ccb-85d3-420dc03811ed, infoPort=46337, infoSecurePort=0, ipcPort=41179, storageInfo=lv=-57;cid=testClusterID;nsid=1955677370;c=1731875024098) 2024-11-17T20:23:47,330 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4742724fdd95ce7a with lease ID 0xeda9b1da7cc5716: from storage DS-96187b8f-658b-441b-a2a5-378169f6917f node DatanodeRegistration(127.0.0.1:38093, datanodeUuid=1404d3bf-c966-4ccb-85d3-420dc03811ed, infoPort=46337, infoSecurePort=0, ipcPort=41179, storageInfo=lv=-57;cid=testClusterID;nsid=1955677370;c=1731875024098), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T20:23:47,330 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4742724fdd95ce7a with lease ID 0xeda9b1da7cc5716: Processing first storage report for DS-0898c31f-73e2-4876-ad03-3e11277b4b23 from datanode DatanodeRegistration(127.0.0.1:38093, datanodeUuid=1404d3bf-c966-4ccb-85d3-420dc03811ed, infoPort=46337, infoSecurePort=0, ipcPort=41179, storageInfo=lv=-57;cid=testClusterID;nsid=1955677370;c=1731875024098) 2024-11-17T20:23:47,330 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4742724fdd95ce7a with lease ID 0xeda9b1da7cc5716: from storage DS-0898c31f-73e2-4876-ad03-3e11277b4b23 node DatanodeRegistration(127.0.0.1:38093, datanodeUuid=1404d3bf-c966-4ccb-85d3-420dc03811ed, infoPort=46337, infoSecurePort=0, ipcPort=41179, storageInfo=lv=-57;cid=testClusterID;nsid=1955677370;c=1731875024098), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T20:23:47,393 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5 2024-11-17T20:23:47,465 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/zookeeper_0, clientPort=60122, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T20:23:47,473 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60122 2024-11-17T20:23:47,483 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T20:23:47,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T20:23:47,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741825_1001 (size=7) 2024-11-17T20:23:48,125 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 with version=8 2024-11-17T20:23:48,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/hbase-staging 2024-11-17T20:23:48,209 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T20:23:48,446 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0b0b0ae8de03:0 server-side Connection retries=45 2024-11-17T20:23:48,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T20:23:48,455 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T20:23:48,460 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T20:23:48,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T20:23:48,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T20:23:48,605 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T20:23:48,664 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T20:23:48,673 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T20:23:48,676 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T20:23:48,700 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33340 (auto-detected) 2024-11-17T20:23:48,701 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T20:23:48,719 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32801 2024-11-17T20:23:48,738 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32801 connecting to ZooKeeper ensemble=127.0.0.1:60122 2024-11-17T20:23:48,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:328010x0, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T20:23:48,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32801-0x1014a77b5a50000 connected 2024-11-17T20:23:48,916 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T20:23:48,918 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T20:23:48,926 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T20:23:48,931 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828, hbase.cluster.distributed=false 2024-11-17T20:23:48,957 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T20:23:48,961 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32801 2024-11-17T20:23:48,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32801 2024-11-17T20:23:48,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32801 2024-11-17T20:23:48,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32801 2024-11-17T20:23:48,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32801 2024-11-17T20:23:49,060 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b0b0ae8de03:0 server-side Connection retries=45 2024-11-17T20:23:49,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T20:23:49,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T20:23:49,062 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T20:23:49,062 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T20:23:49,062 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T20:23:49,064 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T20:23:49,067 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T20:23:49,068 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46555 2024-11-17T20:23:49,070 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46555 connecting to ZooKeeper ensemble=127.0.0.1:60122 2024-11-17T20:23:49,071 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T20:23:49,074 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T20:23:49,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465550x0, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T20:23:49,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:465550x0, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T20:23:49,092 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46555-0x1014a77b5a50001 connected 2024-11-17T20:23:49,095 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T20:23:49,102 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T20:23:49,104 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T20:23:49,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T20:23:49,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46555 2024-11-17T20:23:49,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46555 2024-11-17T20:23:49,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46555 2024-11-17T20:23:49,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46555 2024-11-17T20:23:49,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46555 2024-11-17T20:23:49,125 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b0b0ae8de03:32801 2024-11-17T20:23:49,126 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:49,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T20:23:49,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T20:23:49,137 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:49,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T20:23:49,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:49,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:49,168 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T20:23:49,169 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b0b0ae8de03,32801,1731875028288 from backup master directory 2024-11-17T20:23:49,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T20:23:49,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:49,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T20:23:49,198 WARN [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T20:23:49,198 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:49,201 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T20:23:49,203 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T20:23:49,256 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/hbase.id] with ID: d44a8168-0c44-49be-a02e-28b5ba4ed280 2024-11-17T20:23:49,256 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/.tmp/hbase.id 2024-11-17T20:23:49,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741826_1002 (size=42) 2024-11-17T20:23:49,667 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/.tmp/hbase.id]:[hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/hbase.id] 2024-11-17T20:23:49,724 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T20:23:49,729 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T20:23:49,748 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-17T20:23:49,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:49,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:49,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741827_1003 (size=196) 2024-11-17T20:23:49,837 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:23:49,842 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T20:23:49,863 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:49,869 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T20:23:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741828_1004 (size=1189) 2024-11-17T20:23:49,947 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store 2024-11-17T20:23:49,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741829_1005 (size=34) 2024-11-17T20:23:50,379 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T20:23:50,383 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:23:50,385 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T20:23:50,386 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:23:50,386 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:23:50,388 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T20:23:50,388 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:23:50,389 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:23:50,390 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731875030385Disabling compacts and flushes for region at 1731875030385Disabling writes for close at 1731875030388 (+3 ms)Writing region close event to WAL at 1731875030388Closed at 1731875030388 2024-11-17T20:23:50,394 WARN [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/.initializing 2024-11-17T20:23:50,394 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/WALs/0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:50,406 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T20:23:50,428 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C32801%2C1731875028288, suffix=, logDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/WALs/0b0b0ae8de03,32801,1731875028288, archiveDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/oldWALs, maxLogs=10 2024-11-17T20:23:50,461 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/WALs/0b0b0ae8de03,32801,1731875028288/0b0b0ae8de03%2C32801%2C1731875028288.1731875030435, exclude list is [], retry=0 2024-11-17T20:23:50,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38093,DS-96187b8f-658b-441b-a2a5-378169f6917f,DISK] 2024-11-17T20:23:50,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T20:23:50,533 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/WALs/0b0b0ae8de03,32801,1731875028288/0b0b0ae8de03%2C32801%2C1731875028288.1731875030435 2024-11-17T20:23:50,534 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46337:46337)] 2024-11-17T20:23:50,535 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:23:50,535 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:23:50,538 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,540 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,579 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T20:23:50,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:50,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:50,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T20:23:50,626 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:50,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:23:50,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T20:23:50,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:50,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:23:50,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T20:23:50,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:50,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:23:50,640 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,645 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,647 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,653 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,654 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,658 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T20:23:50,663 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T20:23:50,669 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:23:50,671 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67184228, jitterRate=0.0011230111122131348}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T20:23:50,677 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731875030553Initializing all the Stores at 1731875030555 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875030556 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875030556Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875030557 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875030557Cleaning up temporary data from old regions at 1731875030654 (+97 ms)Region opened successfully at 1731875030677 (+23 ms) 2024-11-17T20:23:50,679 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T20:23:50,714 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be0e44b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0 2024-11-17T20:23:50,746 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T20:23:50,756 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T20:23:50,756 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T20:23:50,758 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T20:23:50,760 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T20:23:50,765 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-17T20:23:50,765 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T20:23:50,794 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T20:23:50,806 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T20:23:50,893 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T20:23:50,896 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T20:23:50,898 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T20:23:50,903 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T20:23:50,906 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T20:23:50,909 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T20:23:50,922 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T20:23:50,924 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T20:23:50,935 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T20:23:50,957 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T20:23:50,967 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T20:23:50,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T20:23:50,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T20:23:50,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:50,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:50,982 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b0b0ae8de03,32801,1731875028288, sessionid=0x1014a77b5a50000, setting cluster-up flag (Was=false) 2024-11-17T20:23:51,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:51,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:51,051 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T20:23:51,053 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:51,103 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T20:23:51,106 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:51,113 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T20:23:51,116 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(746): ClusterId : d44a8168-0c44-49be-a02e-28b5ba4ed280 2024-11-17T20:23:51,118 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T20:23:51,127 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T20:23:51,127 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T20:23:51,137 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T20:23:51,138 DEBUG [RS:0;0b0b0ae8de03:46555 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b17866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b0b0ae8de03/172.17.0.2:0 2024-11-17T20:23:51,157 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b0b0ae8de03:46555 2024-11-17T20:23:51,161 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T20:23:51,161 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T20:23:51,162 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T20:23:51,164 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,32801,1731875028288 with port=46555, startcode=1731875029029 2024-11-17T20:23:51,175 DEBUG [RS:0;0b0b0ae8de03:46555 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T20:23:51,202 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T20:23:51,216 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T20:23:51,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38557, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T20:23:51,226 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T20:23:51,230 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32801 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:51,233 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b0b0ae8de03,32801,1731875028288 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T20:23:51,243 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5 2024-11-17T20:23:51,244 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5 2024-11-17T20:23:51,244 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5 2024-11-17T20:23:51,244 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=5, maxPoolSize=5 2024-11-17T20:23:51,245 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b0b0ae8de03:0, corePoolSize=10, maxPoolSize=10 2024-11-17T20:23:51,245 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,245 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2 2024-11-17T20:23:51,246 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,252 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731875061252 2024-11-17T20:23:51,254 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T20:23:51,254 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T20:23:51,255 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T20:23:51,256 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T20:23:51,258 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T20:23:51,258 WARN [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T20:23:51,260 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T20:23:51,260 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T20:23:51,260 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T20:23:51,261 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T20:23:51,263 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:51,263 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T20:23:51,263 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,266 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T20:23:51,268 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T20:23:51,268 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T20:23:51,270 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T20:23:51,271 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T20:23:51,273 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875031272,5,FailOnTimeoutGroup] 2024-11-17T20:23:51,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741831_1007 (size=1321) 2024-11-17T20:23:51,280 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875031273,5,FailOnTimeoutGroup] 2024-11-17T20:23:51,280 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,281 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T20:23:51,282 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,283 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,360 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b0b0ae8de03,32801,1731875028288 with port=46555, startcode=1731875029029 2024-11-17T20:23:51,362 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32801 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:51,364 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32801 {}] master.ServerManager(517): Registering regionserver=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:51,371 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:23:51,371 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42493 2024-11-17T20:23:51,371 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T20:23:51,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T20:23:51,399 DEBUG [RS:0;0b0b0ae8de03:46555 {}] zookeeper.ZKUtil(111): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:51,399 WARN [RS:0;0b0b0ae8de03:46555 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T20:23:51,400 INFO [RS:0;0b0b0ae8de03:46555 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T20:23:51,400 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/WALs/0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:51,402 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b0b0ae8de03,46555,1731875029029] 2024-11-17T20:23:51,423 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T20:23:51,439 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T20:23:51,444 INFO [RS:0;0b0b0ae8de03:46555 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T20:23:51,444 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,445 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T20:23:51,449 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T20:23:51,451 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,451 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,451 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,451 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,451 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,451 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=2, maxPoolSize=2 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b0b0ae8de03:0, corePoolSize=1, maxPoolSize=1 2024-11-17T20:23:51,452 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3 2024-11-17T20:23:51,453 DEBUG [RS:0;0b0b0ae8de03:46555 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0, corePoolSize=3, maxPoolSize=3 2024-11-17T20:23:51,453 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,454 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,454 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,454 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,454 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,454 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,46555,1731875029029-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T20:23:51,477 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T20:23:51,480 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,46555,1731875029029-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,480 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,481 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.Replication(171): 0b0b0ae8de03,46555,1731875029029 started 2024-11-17T20:23:51,502 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:51,502 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1482): Serving as 0b0b0ae8de03,46555,1731875029029, RpcServer on 0b0b0ae8de03/172.17.0.2:46555, sessionid=0x1014a77b5a50001 2024-11-17T20:23:51,503 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T20:23:51,503 DEBUG [RS:0;0b0b0ae8de03:46555 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:51,504 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,46555,1731875029029' 2024-11-17T20:23:51,504 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T20:23:51,505 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T20:23:51,506 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T20:23:51,506 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T20:23:51,506 DEBUG [RS:0;0b0b0ae8de03:46555 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:51,506 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b0b0ae8de03,46555,1731875029029' 2024-11-17T20:23:51,506 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T20:23:51,507 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T20:23:51,508 DEBUG [RS:0;0b0b0ae8de03:46555 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T20:23:51,508 INFO [RS:0;0b0b0ae8de03:46555 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T20:23:51,508 INFO [RS:0;0b0b0ae8de03:46555 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T20:23:51,615 INFO [RS:0;0b0b0ae8de03:46555 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T20:23:51,619 INFO [RS:0;0b0b0ae8de03:46555 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C46555%2C1731875029029, suffix=, logDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/WALs/0b0b0ae8de03,46555,1731875029029, archiveDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/oldWALs, maxLogs=32 2024-11-17T20:23:51,634 DEBUG [RS:0;0b0b0ae8de03:46555 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/WALs/0b0b0ae8de03,46555,1731875029029/0b0b0ae8de03%2C46555%2C1731875029029.1731875031622, exclude list is [], retry=0 2024-11-17T20:23:51,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38093,DS-96187b8f-658b-441b-a2a5-378169f6917f,DISK] 2024-11-17T20:23:51,643 INFO [RS:0;0b0b0ae8de03:46555 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/WALs/0b0b0ae8de03,46555,1731875029029/0b0b0ae8de03%2C46555%2C1731875029029.1731875031622 2024-11-17T20:23:51,644 DEBUG [RS:0;0b0b0ae8de03:46555 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46337:46337)] 2024-11-17T20:23:51,682 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T20:23:51,683 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:23:51,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741833_1009 (size=32) 2024-11-17T20:23:52,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:23:52,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T20:23:52,106 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T20:23:52,106 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T20:23:52,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T20:23:52,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T20:23:52,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T20:23:52,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T20:23:52,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T20:23:52,122 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T20:23:52,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740 2024-11-17T20:23:52,131 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740 2024-11-17T20:23:52,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T20:23:52,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T20:23:52,134 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:23:52,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T20:23:52,143 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:23:52,144 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67665855, jitterRate=0.0082998126745224}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:23:52,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731875032095Initializing all the Stores at 1731875032097 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875032097Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875032098 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875032098Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875032098Cleaning up temporary data from old regions at 1731875032134 (+36 ms)Region opened successfully at 1731875032148 (+14 ms) 2024-11-17T20:23:52,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T20:23:52,149 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T20:23:52,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T20:23:52,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T20:23:52,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T20:23:52,151 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T20:23:52,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731875032149Disabling compacts and flushes for region at 1731875032149Disabling writes for close at 1731875032149Writing region close event to WAL at 1731875032150 (+1 ms)Closed at 1731875032151 (+1 ms) 2024-11-17T20:23:52,155 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T20:23:52,155 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T20:23:52,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T20:23:52,170 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T20:23:52,173 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T20:23:52,328 DEBUG [0b0b0ae8de03:32801 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T20:23:52,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:52,348 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,46555,1731875029029, state=OPENING 2024-11-17T20:23:52,377 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T20:23:52,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:52,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:23:52,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T20:23:52,388 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T20:23:52,390 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T20:23:52,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:23:52,572 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T20:23:52,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44501, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T20:23:52,587 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T20:23:52,587 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T20:23:52,588 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T20:23:52,592 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b0b0ae8de03%2C46555%2C1731875029029.meta, suffix=.meta, logDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/WALs/0b0b0ae8de03,46555,1731875029029, archiveDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/oldWALs, maxLogs=32 2024-11-17T20:23:52,613 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/WALs/0b0b0ae8de03,46555,1731875029029/0b0b0ae8de03%2C46555%2C1731875029029.meta.1731875032594.meta, exclude list is [], retry=0 2024-11-17T20:23:52,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38093,DS-96187b8f-658b-441b-a2a5-378169f6917f,DISK] 2024-11-17T20:23:52,621 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/WALs/0b0b0ae8de03,46555,1731875029029/0b0b0ae8de03%2C46555%2C1731875029029.meta.1731875032594.meta 2024-11-17T20:23:52,621 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46337:46337)] 2024-11-17T20:23:52,622 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:23:52,623 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T20:23:52,626 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T20:23:52,630 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T20:23:52,634 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T20:23:52,634 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:23:52,634 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T20:23:52,634 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T20:23:52,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T20:23:52,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T20:23:52,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T20:23:52,642 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T20:23:52,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T20:23:52,645 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T20:23:52,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T20:23:52,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T20:23:52,647 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:52,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T20:23:52,649 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T20:23:52,650 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740 2024-11-17T20:23:52,653 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740 2024-11-17T20:23:52,655 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T20:23:52,655 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T20:23:52,656 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:23:52,659 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T20:23:52,661 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74634393, jitterRate=0.11213912069797516}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:23:52,661 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T20:23:52,662 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731875032635Writing region info on filesystem at 1731875032635Initializing all the Stores at 1731875032637 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875032637Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875032637Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875032637Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731875032637Cleaning up temporary data from old regions at 1731875032656 (+19 ms)Running coprocessor post-open hooks at 1731875032661 (+5 ms)Region opened successfully at 1731875032662 (+1 ms) 2024-11-17T20:23:52,668 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731875032562 2024-11-17T20:23:52,678 DEBUG [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T20:23:52,678 INFO [RS_OPEN_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T20:23:52,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:52,683 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b0b0ae8de03,46555,1731875029029, state=OPEN 2024-11-17T20:23:52,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T20:23:52,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T20:23:52,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T20:23:52,732 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T20:23:52,732 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:52,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T20:23:52,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b0b0ae8de03,46555,1731875029029 in 340 msec 2024-11-17T20:23:52,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T20:23:52,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 579 msec 2024-11-17T20:23:52,748 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T20:23:52,748 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T20:23:52,769 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:52,771 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:52,796 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:52,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53145, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:52,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6770 sec 2024-11-17T20:23:52,830 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731875032830, completionTime=-1 2024-11-17T20:23:52,833 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T20:23:52,833 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-17T20:23:52,860 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-17T20:23:52,860 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731875092860 2024-11-17T20:23:52,860 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731875152860 2024-11-17T20:23:52,860 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-17T20:23:52,862 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,32801,1731875028288-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:52,863 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,32801,1731875028288-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:52,863 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,32801,1731875028288-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:52,864 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b0b0ae8de03:32801, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:52,865 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:52,865 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:52,872 DEBUG [master/0b0b0ae8de03:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T20:23:52,893 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.695sec 2024-11-17T20:23:52,894 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T20:23:52,895 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T20:23:52,896 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T20:23:52,896 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T20:23:52,897 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T20:23:52,897 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,32801,1731875028288-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T20:23:52,898 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,32801,1731875028288-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T20:23:52,906 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T20:23:52,907 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T20:23:52,908 INFO [master/0b0b0ae8de03:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b0b0ae8de03,32801,1731875028288-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T20:23:52,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:52,928 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T20:23:52,928 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T20:23:52,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:52,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:52,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42167a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:52,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:52,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:52,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:52,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:52,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:52,996 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=350, ProcessCount=11, AvailableMemoryMB=9475 2024-11-17T20:23:53,023 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T20:23:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@249d22a 2024-11-17T20:23:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T20:23:53,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T20:23:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:23:53,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:23:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T20:23:53,062 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T20:23:53,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T20:23:53,064 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:53,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T20:23:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T20:23:53,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741835_1011 (size=959) 2024-11-17T20:23:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T20:23:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T20:23:53,487 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:23:53,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741836_1012 (size=53) 2024-11-17T20:23:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T20:23:53,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:23:53,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ff04589cbd7acbe489756dc73d4cf811, disabling compactions & flushes 2024-11-17T20:23:53,900 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:53,900 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:53,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. after waiting 0 ms 2024-11-17T20:23:53,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:53,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:53,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff04589cbd7acbe489756dc73d4cf811: Waiting for close lock at 1731875033900Disabling compacts and flushes for region at 1731875033900Disabling writes for close at 1731875033901 (+1 ms)Writing region close event to WAL at 1731875033901Closed at 1731875033901 2024-11-17T20:23:53,905 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T20:23:53,913 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731875033906"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875033906"}]},"ts":"1731875033906"} 2024-11-17T20:23:53,917 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T20:23:53,919 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T20:23:53,921 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875033919"}]},"ts":"1731875033919"} 2024-11-17T20:23:53,925 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T20:23:53,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, ASSIGN}] 2024-11-17T20:23:53,928 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, ASSIGN 2024-11-17T20:23:53,930 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=false 2024-11-17T20:23:54,082 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff04589cbd7acbe489756dc73d4cf811, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:54,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, ASSIGN because future has completed 2024-11-17T20:23:54,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T20:23:54,256 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:54,257 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:23:54,257 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,258 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:23:54,258 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,258 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,261 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,263 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff04589cbd7acbe489756dc73d4cf811 columnFamilyName A 2024-11-17T20:23:54,263 DEBUG [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:54,264 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] regionserver.HStore(327): Store=ff04589cbd7acbe489756dc73d4cf811/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:23:54,265 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,267 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff04589cbd7acbe489756dc73d4cf811 columnFamilyName B 2024-11-17T20:23:54,267 DEBUG [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:54,268 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] regionserver.HStore(327): Store=ff04589cbd7acbe489756dc73d4cf811/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:23:54,268 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,270 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff04589cbd7acbe489756dc73d4cf811 columnFamilyName C 2024-11-17T20:23:54,270 DEBUG [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:23:54,271 INFO [StoreOpener-ff04589cbd7acbe489756dc73d4cf811-1 {}] regionserver.HStore(327): Store=ff04589cbd7acbe489756dc73d4cf811/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:23:54,271 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,273 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,273 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,275 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,275 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,276 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:23:54,279 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,283 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:23:54,284 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ff04589cbd7acbe489756dc73d4cf811; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73179641, jitterRate=0.0904616266489029}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:23:54,284 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:54,285 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ff04589cbd7acbe489756dc73d4cf811: Running coprocessor pre-open hook at 1731875034258Writing region info on filesystem at 1731875034258Initializing all the Stores at 1731875034260 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875034260Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875034261 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875034261Cleaning up temporary data from old regions at 1731875034275 (+14 ms)Running coprocessor post-open hooks at 1731875034284 (+9 ms)Region opened successfully at 1731875034285 (+1 ms) 2024-11-17T20:23:54,287 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., pid=6, masterSystemTime=1731875034246 2024-11-17T20:23:54,290 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:54,290 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:54,291 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff04589cbd7acbe489756dc73d4cf811, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:54,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:23:54,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T20:23:54,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 in 207 msec 2024-11-17T20:23:54,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T20:23:54,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, ASSIGN in 376 msec 2024-11-17T20:23:54,310 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T20:23:54,310 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875034310"}]},"ts":"1731875034310"} 2024-11-17T20:23:54,314 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T20:23:54,317 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T20:23:54,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2670 sec 2024-11-17T20:23:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T20:23:55,232 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:23:55,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,240 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee18a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,245 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fdfb8ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,248 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,256 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T20:23:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,32801,1731875028288 2024-11-17T20:23:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3695840b 2024-11-17T20:23:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T20:23:55,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T20:23:55,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,272 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b390d5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,278 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546221b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,285 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7def95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,290 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c373c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,296 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6488fd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ca9e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b83ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4297415c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,322 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4230eeca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554bc07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,331 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f67bf3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a027360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,339 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b83c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e78fe2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,350 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368cb3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,354 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3714a06f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:23:55,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:23:55,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:23:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:23:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:23:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29536084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:23:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:23:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:23:55,364 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:23:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c6015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:23:55,373 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,374 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,389 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,390 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,392 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,395 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,398 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,403 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,425 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,427 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,427 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:23:55,429 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,429 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:23:55,429 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,429 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:23:55,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:23:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,436 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:23:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:23:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T20:23:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T20:23:55,447 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:23:55,449 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:23:55,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:23:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:55,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:23:55,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:23:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T20:23:55,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/02d5e645eefc475bb494afba99a9f2df is 50, key is test_row_0/A:col10/1731875035491/Put/seqid=0 2024-11-17T20:23:55,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T20:23:55,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:55,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:55,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741837_1013 (size=14341) 2024-11-17T20:23:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875095643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875095643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875095643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875095651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875095638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T20:23:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875095781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875095782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875095782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875095783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875095790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T20:23:55,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:55,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:55,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:55,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T20:23:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:55,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:55,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875095990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:55,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875095992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875096001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875096001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875096010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/02d5e645eefc475bb494afba99a9f2df 2024-11-17T20:23:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T20:23:56,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T20:23:56,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:56,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:56,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/f5b59a587462436e8b8922fffc56071d is 50, key is test_row_0/B:col10/1731875035491/Put/seqid=0 2024-11-17T20:23:56,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741838_1014 (size=12001) 2024-11-17T20:23:56,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/f5b59a587462436e8b8922fffc56071d 2024-11-17T20:23:56,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3349e978ff0f4a028e6733a11a68472e is 50, key is test_row_0/C:col10/1731875035491/Put/seqid=0 2024-11-17T20:23:56,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T20:23:56,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:56,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:56,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741839_1015 (size=12001) 2024-11-17T20:23:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875096301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3349e978ff0f4a028e6733a11a68472e 2024-11-17T20:23:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875096301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875096310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875096322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/02d5e645eefc475bb494afba99a9f2df as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/02d5e645eefc475bb494afba99a9f2df 2024-11-17T20:23:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875096322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/02d5e645eefc475bb494afba99a9f2df, entries=200, sequenceid=13, filesize=14.0 K 2024-11-17T20:23:56,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/f5b59a587462436e8b8922fffc56071d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/f5b59a587462436e8b8922fffc56071d 2024-11-17T20:23:56,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/f5b59a587462436e8b8922fffc56071d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T20:23:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3349e978ff0f4a028e6733a11a68472e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3349e978ff0f4a028e6733a11a68472e 2024-11-17T20:23:56,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3349e978ff0f4a028e6733a11a68472e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T20:23:56,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ff04589cbd7acbe489756dc73d4cf811 in 923ms, sequenceid=13, compaction requested=false 2024-11-17T20:23:56,420 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T20:23:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:56,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T20:23:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:56,460 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:23:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/f8fc460e25d34672a9d270ef04bb6485 is 50, key is test_row_0/A:col10/1731875035635/Put/seqid=0 2024-11-17T20:23:56,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741840_1016 (size=12001) 2024-11-17T20:23:56,499 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/f8fc460e25d34672a9d270ef04bb6485 2024-11-17T20:23:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/486697311638400f91a8be8b0d0e10bb is 50, key is test_row_0/B:col10/1731875035635/Put/seqid=0 2024-11-17T20:23:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T20:23:56,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741841_1017 (size=12001) 2024-11-17T20:23:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:56,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875096860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875096868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875096872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875096876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875096876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875096981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875096982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,986 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/486697311638400f91a8be8b0d0e10bb 2024-11-17T20:23:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:56,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875096991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875096992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875096992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/8bd00c1e40e6469595c5723eeb8441eb is 50, key is test_row_0/C:col10/1731875035635/Put/seqid=0 2024-11-17T20:23:57,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741842_1018 (size=12001) 2024-11-17T20:23:57,045 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/8bd00c1e40e6469595c5723eeb8441eb 2024-11-17T20:23:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/f8fc460e25d34672a9d270ef04bb6485 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/f8fc460e25d34672a9d270ef04bb6485 2024-11-17T20:23:57,090 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/f8fc460e25d34672a9d270ef04bb6485, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T20:23:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/486697311638400f91a8be8b0d0e10bb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/486697311638400f91a8be8b0d0e10bb 2024-11-17T20:23:57,113 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/486697311638400f91a8be8b0d0e10bb, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T20:23:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/8bd00c1e40e6469595c5723eeb8441eb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8bd00c1e40e6469595c5723eeb8441eb 2024-11-17T20:23:57,135 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8bd00c1e40e6469595c5723eeb8441eb, entries=150, sequenceid=37, filesize=11.7 K 2024-11-17T20:23:57,137 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for ff04589cbd7acbe489756dc73d4cf811 in 677ms, sequenceid=37, compaction requested=false 2024-11-17T20:23:57,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:57,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:57,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T20:23:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T20:23:57,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T20:23:57,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6900 sec 2024-11-17T20:23:57,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7160 sec 2024-11-17T20:23:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:23:57,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:23:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:57,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/84d136af65c84a6ab8666471ec64df94 is 50, key is test_row_0/A:col10/1731875037198/Put/seqid=0 2024-11-17T20:23:57,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741843_1019 (size=12001) 2024-11-17T20:23:57,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/84d136af65c84a6ab8666471ec64df94 2024-11-17T20:23:57,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/af5f1892c5c64139b798a3f972b6b91b is 50, key is test_row_0/B:col10/1731875037198/Put/seqid=0 2024-11-17T20:23:57,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875097308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875097311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875097314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875097322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875097323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741844_1020 (size=12001) 2024-11-17T20:23:57,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/af5f1892c5c64139b798a3f972b6b91b 2024-11-17T20:23:57,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/31223b12639d4368b06a2f7863320e24 is 50, key is test_row_0/C:col10/1731875037198/Put/seqid=0 2024-11-17T20:23:57,419 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T20:23:57,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741845_1021 (size=12001) 2024-11-17T20:23:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875097430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875097430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875097430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875097441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875097441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T20:23:57,590 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:23:57,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:23:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T20:23:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T20:23:57,605 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:23:57,607 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:23:57,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:23:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875097639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875097650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875097650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875097660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875097660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T20:23:57,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T20:23:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:57,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:57,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/31223b12639d4368b06a2f7863320e24 2024-11-17T20:23:57,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/84d136af65c84a6ab8666471ec64df94 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/84d136af65c84a6ab8666471ec64df94 2024-11-17T20:23:57,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/84d136af65c84a6ab8666471ec64df94, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T20:23:57,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/af5f1892c5c64139b798a3f972b6b91b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/af5f1892c5c64139b798a3f972b6b91b 2024-11-17T20:23:57,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/af5f1892c5c64139b798a3f972b6b91b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T20:23:57,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/31223b12639d4368b06a2f7863320e24 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31223b12639d4368b06a2f7863320e24 2024-11-17T20:23:57,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31223b12639d4368b06a2f7863320e24, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T20:23:57,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ff04589cbd7acbe489756dc73d4cf811 in 714ms, sequenceid=51, compaction requested=true 2024-11-17T20:23:57,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T20:23:57,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:23:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:23:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:23:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:23:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:23:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:23:57,926 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:23:57,926 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:23:57,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T20:23:57,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:57,929 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T20:23:57,930 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:23:57,932 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:23:57,933 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:57,933 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/f5b59a587462436e8b8922fffc56071d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/486697311638400f91a8be8b0d0e10bb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/af5f1892c5c64139b798a3f972b6b91b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.2 K 2024-11-17T20:23:57,935 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f5b59a587462436e8b8922fffc56071d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731875035422 2024-11-17T20:23:57,936 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:23:57,936 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:23:57,936 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:57,936 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/02d5e645eefc475bb494afba99a9f2df, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/f8fc460e25d34672a9d270ef04bb6485, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/84d136af65c84a6ab8666471ec64df94] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=37.4 K 2024-11-17T20:23:57,937 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 486697311638400f91a8be8b0d0e10bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731875035629 2024-11-17T20:23:57,937 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02d5e645eefc475bb494afba99a9f2df, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731875035422 2024-11-17T20:23:57,938 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting af5f1892c5c64139b798a3f972b6b91b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731875036843 2024-11-17T20:23:57,938 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8fc460e25d34672a9d270ef04bb6485, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731875035629 2024-11-17T20:23:57,939 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84d136af65c84a6ab8666471ec64df94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731875036843 2024-11-17T20:23:57,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b5a9e39a45484c858f03966f0758d6d1 is 50, key is test_row_0/A:col10/1731875037275/Put/seqid=0 2024-11-17T20:23:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:57,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:58,001 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#10 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:23:58,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741846_1022 (size=12001) 2024-11-17T20:23:58,003 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/8d119145a8a949498c600f413a296cd5 is 50, key is test_row_0/B:col10/1731875037198/Put/seqid=0 2024-11-17T20:23:58,004 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b5a9e39a45484c858f03966f0758d6d1 2024-11-17T20:23:58,027 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#11 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:23:58,027 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/c39ae7fc3ae3440bb17efe459c3b843d is 50, key is test_row_0/A:col10/1731875037198/Put/seqid=0 2024-11-17T20:23:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875098024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875098027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875098030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741847_1023 (size=12104) 2024-11-17T20:23:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875098034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875098037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/fbeaf05e07474eecb3fd1c675cff5280 is 50, key is test_row_0/B:col10/1731875037275/Put/seqid=0 2024-11-17T20:23:58,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741848_1024 (size=12104) 2024-11-17T20:23:58,106 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/c39ae7fc3ae3440bb17efe459c3b843d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/c39ae7fc3ae3440bb17efe459c3b843d 2024-11-17T20:23:58,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741849_1025 (size=12001) 2024-11-17T20:23:58,119 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/fbeaf05e07474eecb3fd1c675cff5280 2024-11-17T20:23:58,134 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into c39ae7fc3ae3440bb17efe459c3b843d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:23:58,134 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:58,137 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875037920; duration=0sec 2024-11-17T20:23:58,138 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:23:58,138 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:23:58,138 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:23:58,143 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:23:58,143 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:23:58,143 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:58,144 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3349e978ff0f4a028e6733a11a68472e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8bd00c1e40e6469595c5723eeb8441eb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31223b12639d4368b06a2f7863320e24] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.2 K 2024-11-17T20:23:58,145 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3349e978ff0f4a028e6733a11a68472e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731875035422 2024-11-17T20:23:58,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5802a35baf4a40b0ad3888d8c87f39ec is 50, key is test_row_0/C:col10/1731875037275/Put/seqid=0 2024-11-17T20:23:58,146 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bd00c1e40e6469595c5723eeb8441eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731875035629 2024-11-17T20:23:58,157 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31223b12639d4368b06a2f7863320e24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731875036843 2024-11-17T20:23:58,158 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T20:23:58,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875098150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875098150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875098160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875098160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875098170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741850_1026 (size=12001) 2024-11-17T20:23:58,198 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5802a35baf4a40b0ad3888d8c87f39ec 2024-11-17T20:23:58,215 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#14 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:23:58,215 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/31485b7e597f446a8faacb0de1470d5b is 50, key is test_row_0/C:col10/1731875037198/Put/seqid=0 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T20:23:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b5a9e39a45484c858f03966f0758d6d1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b5a9e39a45484c858f03966f0758d6d1 2024-11-17T20:23:58,249 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b5a9e39a45484c858f03966f0758d6d1, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T20:23:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/fbeaf05e07474eecb3fd1c675cff5280 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/fbeaf05e07474eecb3fd1c675cff5280 2024-11-17T20:23:58,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741851_1027 (size=12104) 2024-11-17T20:23:58,276 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/fbeaf05e07474eecb3fd1c675cff5280, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T20:23:58,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5802a35baf4a40b0ad3888d8c87f39ec as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5802a35baf4a40b0ad3888d8c87f39ec 2024-11-17T20:23:58,290 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/31485b7e597f446a8faacb0de1470d5b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31485b7e597f446a8faacb0de1470d5b 2024-11-17T20:23:58,305 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5802a35baf4a40b0ad3888d8c87f39ec, entries=150, sequenceid=73, filesize=11.7 K 2024-11-17T20:23:58,308 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ff04589cbd7acbe489756dc73d4cf811 in 378ms, sequenceid=73, compaction requested=false 2024-11-17T20:23:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:58,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T20:23:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T20:23:58,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T20:23:58,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 704 msec 2024-11-17T20:23:58,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 717 msec 2024-11-17T20:23:58,332 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into 31485b7e597f446a8faacb0de1470d5b(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:23:58,332 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:58,332 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875037923; duration=0sec 2024-11-17T20:23:58,332 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:23:58,333 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:23:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:23:58,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:23:58,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/3941b17d7f7243e3bc1900deab3a3680 is 50, key is test_row_0/A:col10/1731875038026/Put/seqid=0 2024-11-17T20:23:58,468 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/8d119145a8a949498c600f413a296cd5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/8d119145a8a949498c600f413a296cd5 2024-11-17T20:23:58,487 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into 8d119145a8a949498c600f413a296cd5(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:23:58,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741852_1028 (size=14341) 2024-11-17T20:23:58,488 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:58,488 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875037923; duration=0sec 2024-11-17T20:23:58,488 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:23:58,489 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:23:58,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/3941b17d7f7243e3bc1900deab3a3680 2024-11-17T20:23:58,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/a5c96aa53b1c4ec0a4aaf173a994b7b4 is 50, key is test_row_0/B:col10/1731875038026/Put/seqid=0 2024-11-17T20:23:58,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741853_1029 (size=12001) 2024-11-17T20:23:58,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875098570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875098571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875098574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875098583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875098584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,661 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T20:23:58,661 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T20:23:58,664 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T20:23:58,664 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T20:23:58,665 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T20:23:58,665 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T20:23:58,666 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T20:23:58,666 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T20:23:58,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875098700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875098710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875098710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875098720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875098720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T20:23:58,740 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:23:58,745 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:23:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T20:23:58,749 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:23:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T20:23:58,751 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:23:58,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:23:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T20:23:58,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T20:23:58,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:58,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:58,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:58,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:58,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875098910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875098931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875098932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875098932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875098940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:58,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/a5c96aa53b1c4ec0a4aaf173a994b7b4 2024-11-17T20:23:59,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/aa692720d180433e86d63bb73104f8b2 is 50, key is test_row_0/C:col10/1731875038026/Put/seqid=0 2024-11-17T20:23:59,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741854_1030 (size=12001) 2024-11-17T20:23:59,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T20:23:59,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T20:23:59,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T20:23:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:59,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875099220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875099250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875099250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875099250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875099250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T20:23:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T20:23:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/aa692720d180433e86d63bb73104f8b2 2024-11-17T20:23:59,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/3941b17d7f7243e3bc1900deab3a3680 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/3941b17d7f7243e3bc1900deab3a3680 2024-11-17T20:23:59,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/3941b17d7f7243e3bc1900deab3a3680, entries=200, sequenceid=90, filesize=14.0 K 2024-11-17T20:23:59,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/a5c96aa53b1c4ec0a4aaf173a994b7b4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a5c96aa53b1c4ec0a4aaf173a994b7b4 2024-11-17T20:23:59,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a5c96aa53b1c4ec0a4aaf173a994b7b4, entries=150, sequenceid=90, filesize=11.7 K 2024-11-17T20:23:59,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T20:23:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/aa692720d180433e86d63bb73104f8b2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/aa692720d180433e86d63bb73104f8b2 2024-11-17T20:23:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:23:59,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/aa692720d180433e86d63bb73104f8b2, entries=150, sequenceid=90, filesize=11.7 K 2024-11-17T20:23:59,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for ff04589cbd7acbe489756dc73d4cf811 in 1151ms, sequenceid=90, compaction requested=true 2024-11-17T20:23:59,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:23:59,561 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:23:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:23:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:23:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:23:59,561 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:23:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:23:59,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:23:59,563 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:23:59,563 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:23:59,563 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,564 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/8d119145a8a949498c600f413a296cd5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/fbeaf05e07474eecb3fd1c675cff5280, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a5c96aa53b1c4ec0a4aaf173a994b7b4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.3 K 2024-11-17T20:23:59,564 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:23:59,565 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:23:59,565 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,565 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d119145a8a949498c600f413a296cd5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731875036843 2024-11-17T20:23:59,565 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/c39ae7fc3ae3440bb17efe459c3b843d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b5a9e39a45484c858f03966f0758d6d1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/3941b17d7f7243e3bc1900deab3a3680] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=37.5 K 2024-11-17T20:23:59,567 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting fbeaf05e07474eecb3fd1c675cff5280, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731875037275 2024-11-17T20:23:59,567 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c39ae7fc3ae3440bb17efe459c3b843d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731875036843 2024-11-17T20:23:59,567 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c96aa53b1c4ec0a4aaf173a994b7b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731875038026 2024-11-17T20:23:59,568 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a9e39a45484c858f03966f0758d6d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731875037275 2024-11-17T20:23:59,569 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3941b17d7f7243e3bc1900deab3a3680, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731875038023 2024-11-17T20:23:59,607 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:23:59,608 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b200b5314649442ca3ee5b9951866f26 is 50, key is test_row_0/A:col10/1731875038026/Put/seqid=0 2024-11-17T20:23:59,612 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:23:59,613 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/9fbaadbb2f254062838a0fba90428f02 is 50, key is test_row_0/B:col10/1731875038026/Put/seqid=0 2024-11-17T20:23:59,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741855_1031 (size=12207) 2024-11-17T20:23:59,647 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b200b5314649442ca3ee5b9951866f26 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b200b5314649442ca3ee5b9951866f26 2024-11-17T20:23:59,662 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into b200b5314649442ca3ee5b9951866f26(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:23:59,663 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:59,663 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875039561; duration=0sec 2024-11-17T20:23:59,663 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:23:59,663 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:23:59,663 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:23:59,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741856_1032 (size=12207) 2024-11-17T20:23:59,667 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:23:59,667 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:23:59,667 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,668 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31485b7e597f446a8faacb0de1470d5b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5802a35baf4a40b0ad3888d8c87f39ec, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/aa692720d180433e86d63bb73104f8b2] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.3 K 2024-11-17T20:23:59,669 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31485b7e597f446a8faacb0de1470d5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731875036843 2024-11-17T20:23:59,670 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5802a35baf4a40b0ad3888d8c87f39ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731875037275 2024-11-17T20:23:59,671 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa692720d180433e86d63bb73104f8b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731875038026 2024-11-17T20:23:59,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T20:23:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,693 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T20:23:59,711 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:23:59,712 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/9f965244273c4f27a14fe50d9839e4a2 is 50, key is test_row_0/C:col10/1731875038026/Put/seqid=0 2024-11-17T20:23:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/ea8641cf42b1415ca5d3e98088f9897d is 50, key is test_row_0/A:col10/1731875038574/Put/seqid=0 2024-11-17T20:23:59,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741857_1033 (size=12207) 2024-11-17T20:23:59,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:23:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:23:59,750 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/9f965244273c4f27a14fe50d9839e4a2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9f965244273c4f27a14fe50d9839e4a2 2024-11-17T20:23:59,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741858_1034 (size=12001) 2024-11-17T20:23:59,757 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/ea8641cf42b1415ca5d3e98088f9897d 2024-11-17T20:23:59,765 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into 9f965244273c4f27a14fe50d9839e4a2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:23:59,765 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:59,765 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875039561; duration=0sec 2024-11-17T20:23:59,765 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:23:59,765 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:23:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/cb8afd750d4d4d1da713e90297d104a6 is 50, key is test_row_0/B:col10/1731875038574/Put/seqid=0 2024-11-17T20:23:59,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875099789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875099799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875099799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741859_1035 (size=12001) 2024-11-17T20:23:59,817 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/cb8afd750d4d4d1da713e90297d104a6 2024-11-17T20:23:59,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/9765d18fc72b4f53a3d952738ad80741 is 50, key is test_row_0/C:col10/1731875038574/Put/seqid=0 2024-11-17T20:23:59,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741860_1036 (size=12001) 2024-11-17T20:23:59,852 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/9765d18fc72b4f53a3d952738ad80741 2024-11-17T20:23:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/ea8641cf42b1415ca5d3e98088f9897d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea8641cf42b1415ca5d3e98088f9897d 2024-11-17T20:23:59,880 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea8641cf42b1415ca5d3e98088f9897d, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T20:23:59,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/cb8afd750d4d4d1da713e90297d104a6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/cb8afd750d4d4d1da713e90297d104a6 2024-11-17T20:23:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T20:23:59,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875099910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875099910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,916 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/cb8afd750d4d4d1da713e90297d104a6, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T20:23:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875099911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875099910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:23:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875099914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:23:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:23:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:23:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/9765d18fc72b4f53a3d952738ad80741 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9765d18fc72b4f53a3d952738ad80741 2024-11-17T20:23:59,931 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9765d18fc72b4f53a3d952738ad80741, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T20:23:59,934 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ff04589cbd7acbe489756dc73d4cf811 in 241ms, sequenceid=114, compaction requested=false 2024-11-17T20:23:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:23:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:23:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T20:23:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T20:23:59,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T20:23:59,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1850 sec 2024-11-17T20:23:59,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1990 sec 2024-11-17T20:24:00,078 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/9fbaadbb2f254062838a0fba90428f02 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/9fbaadbb2f254062838a0fba90428f02 2024-11-17T20:24:00,093 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into 9fbaadbb2f254062838a0fba90428f02(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:00,093 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:00,093 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875039561; duration=0sec 2024-11-17T20:24:00,094 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:00,094 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:00,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:00,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:24:00,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2 is 50, key is test_row_0/A:col10/1731875040124/Put/seqid=0 2024-11-17T20:24:00,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875100161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875100163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875100167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875100167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875100171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741861_1037 (size=12051) 2024-11-17T20:24:00,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2 2024-11-17T20:24:00,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/43ecf894cd10411c9475cb6d8dcee802 is 50, key is test_row_0/B:col10/1731875040124/Put/seqid=0 2024-11-17T20:24:00,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741862_1038 (size=12051) 2024-11-17T20:24:00,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/43ecf894cd10411c9475cb6d8dcee802 2024-11-17T20:24:00,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5d75a8a6eb2a43a09640adf4c795731e is 50, key is test_row_0/C:col10/1731875040124/Put/seqid=0 2024-11-17T20:24:00,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875100279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875100279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875100280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875100280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875100280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741863_1039 (size=12051) 2024-11-17T20:24:00,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5d75a8a6eb2a43a09640adf4c795731e 2024-11-17T20:24:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2 2024-11-17T20:24:00,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T20:24:00,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/43ecf894cd10411c9475cb6d8dcee802 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/43ecf894cd10411c9475cb6d8dcee802 2024-11-17T20:24:00,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/43ecf894cd10411c9475cb6d8dcee802, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T20:24:00,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5d75a8a6eb2a43a09640adf4c795731e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5d75a8a6eb2a43a09640adf4c795731e 2024-11-17T20:24:00,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5d75a8a6eb2a43a09640adf4c795731e, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T20:24:00,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ff04589cbd7acbe489756dc73d4cf811 in 260ms, sequenceid=133, compaction requested=true 2024-11-17T20:24:00,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:00,387 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:00,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:00,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:00,388 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:00,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:00,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:00,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:00,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:00,389 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:00,389 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:00,389 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:00,390 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b200b5314649442ca3ee5b9951866f26, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea8641cf42b1415ca5d3e98088f9897d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.4 K 2024-11-17T20:24:00,391 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:00,391 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:00,391 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:00,391 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/9fbaadbb2f254062838a0fba90428f02, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/cb8afd750d4d4d1da713e90297d104a6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/43ecf894cd10411c9475cb6d8dcee802] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.4 K 2024-11-17T20:24:00,392 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b200b5314649442ca3ee5b9951866f26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731875038026 2024-11-17T20:24:00,392 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbaadbb2f254062838a0fba90428f02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731875038026 2024-11-17T20:24:00,393 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea8641cf42b1415ca5d3e98088f9897d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731875038558 2024-11-17T20:24:00,394 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8afd750d4d4d1da713e90297d104a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731875038558 2024-11-17T20:24:00,394 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbd8a3ab77a54f0c961a8fb1cd5d35a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731875039795 2024-11-17T20:24:00,396 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ecf894cd10411c9475cb6d8dcee802, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731875039795 2024-11-17T20:24:00,418 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:00,419 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/118103c182fa460a8db46a9a16db7b65 is 50, key is test_row_0/B:col10/1731875040124/Put/seqid=0 2024-11-17T20:24:00,435 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:00,435 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/d2d3f57d25ea4576af47e658dd4c2ef0 is 50, key is test_row_0/A:col10/1731875040124/Put/seqid=0 2024-11-17T20:24:00,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741864_1040 (size=12359) 2024-11-17T20:24:00,486 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/118103c182fa460a8db46a9a16db7b65 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/118103c182fa460a8db46a9a16db7b65 2024-11-17T20:24:00,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741865_1041 (size=12359) 2024-11-17T20:24:00,501 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into 118103c182fa460a8db46a9a16db7b65(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:00,501 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:00,502 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875040388; duration=0sec 2024-11-17T20:24:00,502 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:00,502 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:00,502 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:00,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:00,505 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/d2d3f57d25ea4576af47e658dd4c2ef0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d2d3f57d25ea4576af47e658dd4c2ef0 2024-11-17T20:24:00,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T20:24:00,506 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:00,506 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:00,506 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:00,506 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9f965244273c4f27a14fe50d9839e4a2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9765d18fc72b4f53a3d952738ad80741, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5d75a8a6eb2a43a09640adf4c795731e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.4 K 2024-11-17T20:24:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:00,508 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f965244273c4f27a14fe50d9839e4a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731875038026 2024-11-17T20:24:00,509 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9765d18fc72b4f53a3d952738ad80741, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731875038558 2024-11-17T20:24:00,511 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d75a8a6eb2a43a09640adf4c795731e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731875039795 2024-11-17T20:24:00,518 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into d2d3f57d25ea4576af47e658dd4c2ef0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:00,518 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:00,518 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875040387; duration=0sec 2024-11-17T20:24:00,518 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:00,519 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:00,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/7f0bf5d5fd954f7da35c7d542664aeb1 is 50, key is test_row_0/A:col10/1731875040500/Put/seqid=0 2024-11-17T20:24:00,539 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:00,539 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/cfb57867118244c0ab2607661e7257cb is 50, key is test_row_0/C:col10/1731875040124/Put/seqid=0 2024-11-17T20:24:00,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741866_1042 (size=12151) 2024-11-17T20:24:00,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/7f0bf5d5fd954f7da35c7d542664aeb1 2024-11-17T20:24:00,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875100542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875100550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875100552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875100556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875100556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741867_1043 (size=12359) 2024-11-17T20:24:00,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/52dcf74af15e4984881aab6a5bc5a96b is 50, key is test_row_0/B:col10/1731875040500/Put/seqid=0 2024-11-17T20:24:00,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741868_1044 (size=12151) 2024-11-17T20:24:00,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/52dcf74af15e4984881aab6a5bc5a96b 2024-11-17T20:24:00,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/e31eea5fd0d3444ea59d5361a07a4dd4 is 50, key is test_row_0/C:col10/1731875040500/Put/seqid=0 2024-11-17T20:24:00,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741869_1045 (size=12151) 2024-11-17T20:24:00,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/e31eea5fd0d3444ea59d5361a07a4dd4 2024-11-17T20:24:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875100660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/7f0bf5d5fd954f7da35c7d542664aeb1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/7f0bf5d5fd954f7da35c7d542664aeb1 2024-11-17T20:24:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875100669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875100670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875100670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/7f0bf5d5fd954f7da35c7d542664aeb1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T20:24:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875100672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/52dcf74af15e4984881aab6a5bc5a96b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/52dcf74af15e4984881aab6a5bc5a96b 2024-11-17T20:24:00,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/52dcf74af15e4984881aab6a5bc5a96b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T20:24:00,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/e31eea5fd0d3444ea59d5361a07a4dd4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e31eea5fd0d3444ea59d5361a07a4dd4 2024-11-17T20:24:00,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e31eea5fd0d3444ea59d5361a07a4dd4, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T20:24:00,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ff04589cbd7acbe489756dc73d4cf811 in 202ms, sequenceid=156, compaction requested=false 2024-11-17T20:24:00,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:00,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:00,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:24:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T20:24:00,901 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:00,902 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T20:24:00,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/9007d52809614281a90ead46ffacb61c is 50, key is test_row_0/A:col10/1731875040874/Put/seqid=0 2024-11-17T20:24:00,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T20:24:00,912 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:00,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:00,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875100923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875100923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741870_1046 (size=14541) 2024-11-17T20:24:00,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875100925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875100927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875100930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:00,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/9007d52809614281a90ead46ffacb61c 2024-11-17T20:24:00,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3d3ad21d1a004b29b96db234f9b214cf is 50, key is test_row_0/B:col10/1731875040874/Put/seqid=0 2024-11-17T20:24:00,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741871_1047 (size=12151) 2024-11-17T20:24:00,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3d3ad21d1a004b29b96db234f9b214cf 2024-11-17T20:24:00,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3e213d1b1fad466f9cdd1937641467d2 is 50, key is test_row_0/C:col10/1731875040874/Put/seqid=0 2024-11-17T20:24:01,005 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/cfb57867118244c0ab2607661e7257cb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/cfb57867118244c0ab2607661e7257cb 2024-11-17T20:24:01,016 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into cfb57867118244c0ab2607661e7257cb(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:01,016 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:01,016 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875040388; duration=0sec 2024-11-17T20:24:01,016 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:01,016 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T20:24:01,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741872_1048 (size=12151) 2024-11-17T20:24:01,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875101039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875101039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875101040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875101040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875101040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T20:24:01,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:01,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T20:24:01,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T20:24:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875101249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875101250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875101250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875101250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875101250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T20:24:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:01,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3e213d1b1fad466f9cdd1937641467d2 2024-11-17T20:24:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/9007d52809614281a90ead46ffacb61c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9007d52809614281a90ead46ffacb61c 2024-11-17T20:24:01,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9007d52809614281a90ead46ffacb61c, entries=200, sequenceid=172, filesize=14.2 K 2024-11-17T20:24:01,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3d3ad21d1a004b29b96db234f9b214cf as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3d3ad21d1a004b29b96db234f9b214cf 2024-11-17T20:24:01,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3d3ad21d1a004b29b96db234f9b214cf, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T20:24:01,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3e213d1b1fad466f9cdd1937641467d2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3e213d1b1fad466f9cdd1937641467d2 2024-11-17T20:24:01,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3e213d1b1fad466f9cdd1937641467d2, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T20:24:01,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for ff04589cbd7acbe489756dc73d4cf811 in 608ms, sequenceid=172, compaction requested=true 2024-11-17T20:24:01,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:01,485 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:01,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:01,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:01,486 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:01,488 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:01,488 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:01,488 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:01,488 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:01,488 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,488 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,489 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/118103c182fa460a8db46a9a16db7b65, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/52dcf74af15e4984881aab6a5bc5a96b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3d3ad21d1a004b29b96db234f9b214cf] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.8 K 2024-11-17T20:24:01,489 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d2d3f57d25ea4576af47e658dd4c2ef0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/7f0bf5d5fd954f7da35c7d542664aeb1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9007d52809614281a90ead46ffacb61c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=38.1 K 2024-11-17T20:24:01,489 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 118103c182fa460a8db46a9a16db7b65, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731875039795 2024-11-17T20:24:01,489 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d3f57d25ea4576af47e658dd4c2ef0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731875039795 2024-11-17T20:24:01,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:01,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:01,490 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 52dcf74af15e4984881aab6a5bc5a96b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731875040158 2024-11-17T20:24:01,490 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0bf5d5fd954f7da35c7d542664aeb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731875040158 2024-11-17T20:24:01,490 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3ad21d1a004b29b96db234f9b214cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875040539 2024-11-17T20:24:01,491 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9007d52809614281a90ead46ffacb61c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875040539 2024-11-17T20:24:01,526 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:01,526 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#36 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:01,527 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/4b70c9e722e342948c7f0ba77bc9cb72 is 50, key is test_row_0/B:col10/1731875040874/Put/seqid=0 2024-11-17T20:24:01,530 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/12c7060653164c0aa24bc3598fd3ff15 is 50, key is test_row_0/A:col10/1731875040874/Put/seqid=0 2024-11-17T20:24:01,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T20:24:01,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,538 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T20:24:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T20:24:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/a68eefd457144697b3cd8616a8005277 is 50, key is test_row_0/A:col10/1731875040925/Put/seqid=0 2024-11-17T20:24:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:01,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741873_1049 (size=12561) 2024-11-17T20:24:01,581 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/4b70c9e722e342948c7f0ba77bc9cb72 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/4b70c9e722e342948c7f0ba77bc9cb72 2024-11-17T20:24:01,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741874_1050 (size=12561) 2024-11-17T20:24:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875101585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875101585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875101587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875101589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,601 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/12c7060653164c0aa24bc3598fd3ff15 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/12c7060653164c0aa24bc3598fd3ff15 2024-11-17T20:24:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875101591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,606 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into 4b70c9e722e342948c7f0ba77bc9cb72(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:01,606 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:01,607 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875041486; duration=0sec 2024-11-17T20:24:01,608 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:01,608 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:01,608 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:01,611 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:01,612 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:01,612 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,612 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/cfb57867118244c0ab2607661e7257cb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e31eea5fd0d3444ea59d5361a07a4dd4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3e213d1b1fad466f9cdd1937641467d2] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=35.8 K 2024-11-17T20:24:01,613 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb57867118244c0ab2607661e7257cb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731875039795 2024-11-17T20:24:01,614 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e31eea5fd0d3444ea59d5361a07a4dd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731875040158 2024-11-17T20:24:01,614 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into 12c7060653164c0aa24bc3598fd3ff15(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:01,614 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:01,615 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875041485; duration=0sec 2024-11-17T20:24:01,615 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:01,615 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:01,615 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e213d1b1fad466f9cdd1937641467d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875040539 2024-11-17T20:24:01,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741875_1051 (size=12151) 2024-11-17T20:24:01,623 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/a68eefd457144697b3cd8616a8005277 2024-11-17T20:24:01,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/b14c99a733444d0ca12a3a5891d6f0dd is 50, key is test_row_0/B:col10/1731875040925/Put/seqid=0 2024-11-17T20:24:01,643 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:01,644 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/d92d42405378412d8ee87395b07f89d1 is 50, key is test_row_0/C:col10/1731875040874/Put/seqid=0 2024-11-17T20:24:01,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741877_1053 (size=12561) 2024-11-17T20:24:01,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741876_1052 (size=12151) 2024-11-17T20:24:01,665 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/b14c99a733444d0ca12a3a5891d6f0dd 2024-11-17T20:24:01,673 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/d92d42405378412d8ee87395b07f89d1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/d92d42405378412d8ee87395b07f89d1 2024-11-17T20:24:01,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/6a851775750741c9be822ac326e4ca00 is 50, key is test_row_0/C:col10/1731875040925/Put/seqid=0 2024-11-17T20:24:01,689 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into d92d42405378412d8ee87395b07f89d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:01,689 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:01,689 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875041489; duration=0sec 2024-11-17T20:24:01,689 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:01,689 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:01,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741878_1054 (size=12151) 2024-11-17T20:24:01,694 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/6a851775750741c9be822ac326e4ca00 2024-11-17T20:24:01,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875101700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875101700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875101701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/a68eefd457144697b3cd8616a8005277 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/a68eefd457144697b3cd8616a8005277 2024-11-17T20:24:01,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875101710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875101710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,718 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/a68eefd457144697b3cd8616a8005277, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T20:24:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/b14c99a733444d0ca12a3a5891d6f0dd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b14c99a733444d0ca12a3a5891d6f0dd 2024-11-17T20:24:01,727 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b14c99a733444d0ca12a3a5891d6f0dd, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T20:24:01,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/6a851775750741c9be822ac326e4ca00 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/6a851775750741c9be822ac326e4ca00 2024-11-17T20:24:01,737 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/6a851775750741c9be822ac326e4ca00, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T20:24:01,739 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for ff04589cbd7acbe489756dc73d4cf811 in 202ms, sequenceid=195, compaction requested=false 2024-11-17T20:24:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T20:24:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T20:24:01,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T20:24:01,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 830 msec 2024-11-17T20:24:01,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 847 msec 2024-11-17T20:24:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:01,913 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:01,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T20:24:01,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b8fb03b636464cb78d18aa331c8a05b6 is 50, key is test_row_0/A:col10/1731875041912/Put/seqid=0 2024-11-17T20:24:01,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741879_1055 (size=12151) 2024-11-17T20:24:01,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b8fb03b636464cb78d18aa331c8a05b6 2024-11-17T20:24:01,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3849d4b96060442a9a158da875ca8d87 is 50, key is test_row_0/B:col10/1731875041912/Put/seqid=0 2024-11-17T20:24:01,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875101947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875101948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875101951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875101954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875101957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:01,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741880_1056 (size=12151) 2024-11-17T20:24:01,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3849d4b96060442a9a158da875ca8d87 2024-11-17T20:24:01,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/a8430b5086644a90bfa1ec11ab8a4298 is 50, key is test_row_0/C:col10/1731875041912/Put/seqid=0 2024-11-17T20:24:02,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741881_1057 (size=12151) 2024-11-17T20:24:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T20:24:02,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:02,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T20:24:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T20:24:02,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:02,055 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:02,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875102060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875102060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875102070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875102070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875102070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T20:24:02,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T20:24:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:02,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875102269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875102269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875102280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875102280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875102281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T20:24:02,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:02,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T20:24:02,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/a8430b5086644a90bfa1ec11ab8a4298 2024-11-17T20:24:02,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b8fb03b636464cb78d18aa331c8a05b6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8fb03b636464cb78d18aa331c8a05b6 2024-11-17T20:24:02,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8fb03b636464cb78d18aa331c8a05b6, entries=150, sequenceid=216, filesize=11.9 K 2024-11-17T20:24:02,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3849d4b96060442a9a158da875ca8d87 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3849d4b96060442a9a158da875ca8d87 2024-11-17T20:24:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3849d4b96060442a9a158da875ca8d87, entries=150, sequenceid=216, filesize=11.9 K 2024-11-17T20:24:02,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/a8430b5086644a90bfa1ec11ab8a4298 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a8430b5086644a90bfa1ec11ab8a4298 2024-11-17T20:24:02,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a8430b5086644a90bfa1ec11ab8a4298, entries=150, sequenceid=216, filesize=11.9 K 2024-11-17T20:24:02,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for ff04589cbd7acbe489756dc73d4cf811 in 539ms, sequenceid=216, compaction requested=true 2024-11-17T20:24:02,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:02,453 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:02,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:02,455 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:02,455 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:02,455 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,455 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/12c7060653164c0aa24bc3598fd3ff15, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/a68eefd457144697b3cd8616a8005277, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8fb03b636464cb78d18aa331c8a05b6] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.0 K 2024-11-17T20:24:02,456 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:02,456 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12c7060653164c0aa24bc3598fd3ff15, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875040539 2024-11-17T20:24:02,457 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a68eefd457144697b3cd8616a8005277, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731875040912 2024-11-17T20:24:02,458 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:02,458 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8fb03b636464cb78d18aa331c8a05b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731875041579 2024-11-17T20:24:02,458 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:02,458 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,459 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/4b70c9e722e342948c7f0ba77bc9cb72, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b14c99a733444d0ca12a3a5891d6f0dd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3849d4b96060442a9a158da875ca8d87] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.0 K 2024-11-17T20:24:02,459 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b70c9e722e342948c7f0ba77bc9cb72, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875040539 2024-11-17T20:24:02,460 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting b14c99a733444d0ca12a3a5891d6f0dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731875040912 2024-11-17T20:24:02,461 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3849d4b96060442a9a158da875ca8d87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731875041579 2024-11-17T20:24:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:02,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:02,475 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:02,476 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:02,476 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/5f921df19c014eb0bdacfb8cc670bc36 is 50, key is test_row_0/A:col10/1731875041912/Put/seqid=0 2024-11-17T20:24:02,476 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/bd11612c6b37442887425d849f70bb4c is 50, key is test_row_0/B:col10/1731875041912/Put/seqid=0 2024-11-17T20:24:02,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741883_1059 (size=12663) 2024-11-17T20:24:02,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741882_1058 (size=12663) 2024-11-17T20:24:02,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T20:24:02,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,521 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T20:24:02,524 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/bd11612c6b37442887425d849f70bb4c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/bd11612c6b37442887425d849f70bb4c 2024-11-17T20:24:02,528 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/5f921df19c014eb0bdacfb8cc670bc36 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/5f921df19c014eb0bdacfb8cc670bc36 2024-11-17T20:24:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/89cc054fa09c4335b89ce7b46c2492ab is 50, key is test_row_0/A:col10/1731875041941/Put/seqid=0 2024-11-17T20:24:02,541 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into bd11612c6b37442887425d849f70bb4c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:02,541 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:02,541 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875042455; duration=0sec 2024-11-17T20:24:02,541 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into 5f921df19c014eb0bdacfb8cc670bc36(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:02,541 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:02,541 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:02,541 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:02,541 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875042453; duration=0sec 2024-11-17T20:24:02,541 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:02,541 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:02,541 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:02,545 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:02,545 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:02,546 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:02,546 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/d92d42405378412d8ee87395b07f89d1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/6a851775750741c9be822ac326e4ca00, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a8430b5086644a90bfa1ec11ab8a4298] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.0 K 2024-11-17T20:24:02,546 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d92d42405378412d8ee87395b07f89d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875040539 2024-11-17T20:24:02,547 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a851775750741c9be822ac326e4ca00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731875040912 2024-11-17T20:24:02,548 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a8430b5086644a90bfa1ec11ab8a4298, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731875041579 2024-11-17T20:24:02,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741884_1060 (size=12151) 2024-11-17T20:24:02,574 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:02,575 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3725108eb528419e89ea56546edcedae is 50, key is test_row_0/C:col10/1731875041912/Put/seqid=0 2024-11-17T20:24:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:02,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:02,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741885_1061 (size=12663) 2024-11-17T20:24:02,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875102607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875102610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875102609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875102612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875102616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,626 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3725108eb528419e89ea56546edcedae as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3725108eb528419e89ea56546edcedae 2024-11-17T20:24:02,641 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into 3725108eb528419e89ea56546edcedae(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:02,641 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:02,642 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875042463; duration=0sec 2024-11-17T20:24:02,642 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:02,642 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T20:24:02,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875102719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875102720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875102720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875102729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875102729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875102930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875102930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875102930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875102939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875102940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:02,970 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/89cc054fa09c4335b89ce7b46c2492ab 2024-11-17T20:24:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/69df0b2fd95d4f0cba2afebae5bcf93a is 50, key is test_row_0/B:col10/1731875041941/Put/seqid=0 2024-11-17T20:24:02,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741886_1062 (size=12151) 2024-11-17T20:24:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T20:24:03,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875103239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875103239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875103239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875103250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875103250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,387 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/69df0b2fd95d4f0cba2afebae5bcf93a 2024-11-17T20:24:03,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/261b286680ba453082a73ca475560d69 is 50, key is test_row_0/C:col10/1731875041941/Put/seqid=0 2024-11-17T20:24:03,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741887_1063 (size=12151) 2024-11-17T20:24:03,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875103750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875103750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875103760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875103759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875103769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:03,808 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/261b286680ba453082a73ca475560d69 2024-11-17T20:24:03,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/89cc054fa09c4335b89ce7b46c2492ab as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/89cc054fa09c4335b89ce7b46c2492ab 2024-11-17T20:24:03,821 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/89cc054fa09c4335b89ce7b46c2492ab, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T20:24:03,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/69df0b2fd95d4f0cba2afebae5bcf93a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/69df0b2fd95d4f0cba2afebae5bcf93a 2024-11-17T20:24:03,829 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/69df0b2fd95d4f0cba2afebae5bcf93a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T20:24:03,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/261b286680ba453082a73ca475560d69 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/261b286680ba453082a73ca475560d69 2024-11-17T20:24:03,837 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/261b286680ba453082a73ca475560d69, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T20:24:03,839 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ff04589cbd7acbe489756dc73d4cf811 in 1317ms, sequenceid=235, compaction requested=false 2024-11-17T20:24:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T20:24:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T20:24:03,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T20:24:03,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-11-17T20:24:03,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7950 sec 2024-11-17T20:24:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T20:24:04,210 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:04,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T20:24:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T20:24:04,217 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:04,219 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:04,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T20:24:04,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T20:24:04,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:04,374 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T20:24:04,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/9979bd74fedf4b8d8fadbbb7a60158f0 is 50, key is test_row_0/A:col10/1731875042611/Put/seqid=0 2024-11-17T20:24:04,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741888_1064 (size=12151) 2024-11-17T20:24:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T20:24:04,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:04,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875104789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875104792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875104793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875104794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875104799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T20:24:04,860 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/9979bd74fedf4b8d8fadbbb7a60158f0 2024-11-17T20:24:04,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/794a035b909544ffba1cf3921bfe8412 is 50, key is test_row_0/B:col10/1731875042611/Put/seqid=0 2024-11-17T20:24:04,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741889_1065 (size=12151) 2024-11-17T20:24:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875104909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875104910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875104910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875104910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875104911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:04,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875105120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875105120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875105120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875105120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875105120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,277 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/794a035b909544ffba1cf3921bfe8412 2024-11-17T20:24:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/1cc8a14679ac41c4b0c4f3ab36a1444d is 50, key is test_row_0/C:col10/1731875042611/Put/seqid=0 2024-11-17T20:24:05,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741890_1066 (size=12151) 2024-11-17T20:24:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T20:24:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875105429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875105430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875105430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875105430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875105431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,703 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/1cc8a14679ac41c4b0c4f3ab36a1444d 2024-11-17T20:24:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/9979bd74fedf4b8d8fadbbb7a60158f0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9979bd74fedf4b8d8fadbbb7a60158f0 2024-11-17T20:24:05,723 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9979bd74fedf4b8d8fadbbb7a60158f0, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T20:24:05,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/794a035b909544ffba1cf3921bfe8412 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/794a035b909544ffba1cf3921bfe8412 2024-11-17T20:24:05,732 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/794a035b909544ffba1cf3921bfe8412, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T20:24:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/1cc8a14679ac41c4b0c4f3ab36a1444d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/1cc8a14679ac41c4b0c4f3ab36a1444d 2024-11-17T20:24:05,745 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/1cc8a14679ac41c4b0c4f3ab36a1444d, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T20:24:05,748 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ff04589cbd7acbe489756dc73d4cf811 in 1375ms, sequenceid=255, compaction requested=true 2024-11-17T20:24:05,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:05,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:05,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T20:24:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T20:24:05,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T20:24:05,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5320 sec 2024-11-17T20:24:05,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5490 sec 2024-11-17T20:24:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:05,942 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:05,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T20:24:05,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/28c80648266c4c54b7cf52e49d234563 is 50, key is test_row_0/A:col10/1731875044796/Put/seqid=0 2024-11-17T20:24:05,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741891_1067 (size=14741) 2024-11-17T20:24:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875105961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875105963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875105965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875105966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875105967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875106080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875106080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875106081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875106081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875106081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875106289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875106289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875106289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875106289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875106290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T20:24:06,360 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:06,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T20:24:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T20:24:06,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:06,364 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:06,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:06,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/28c80648266c4c54b7cf52e49d234563 2024-11-17T20:24:06,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3f0086b74c8e4092a8a9cb2b25079b44 is 50, key is test_row_0/B:col10/1731875044796/Put/seqid=0 2024-11-17T20:24:06,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741892_1068 (size=12301) 2024-11-17T20:24:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T20:24:06,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T20:24:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875106600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875106601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875106601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875106601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875106601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:06,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T20:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T20:24:06,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3f0086b74c8e4092a8a9cb2b25079b44 2024-11-17T20:24:06,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/f51850142a0e42e59c683b04c37b5cdf is 50, key is test_row_0/C:col10/1731875044796/Put/seqid=0 2024-11-17T20:24:06,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741893_1069 (size=12301) 2024-11-17T20:24:06,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T20:24:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T20:24:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:06,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:06,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T20:24:07,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875107109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:07,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875107109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875107109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:07,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:07,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875107110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875107110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:07,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T20:24:07,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:07,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:07,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:07,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:07,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:07,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/f51850142a0e42e59c683b04c37b5cdf 2024-11-17T20:24:07,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/28c80648266c4c54b7cf52e49d234563 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/28c80648266c4c54b7cf52e49d234563 2024-11-17T20:24:07,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/28c80648266c4c54b7cf52e49d234563, entries=200, sequenceid=276, filesize=14.4 K 2024-11-17T20:24:07,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/3f0086b74c8e4092a8a9cb2b25079b44 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3f0086b74c8e4092a8a9cb2b25079b44 2024-11-17T20:24:07,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3f0086b74c8e4092a8a9cb2b25079b44, entries=150, sequenceid=276, filesize=12.0 K 2024-11-17T20:24:07,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/f51850142a0e42e59c683b04c37b5cdf as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/f51850142a0e42e59c683b04c37b5cdf 2024-11-17T20:24:07,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/f51850142a0e42e59c683b04c37b5cdf, entries=150, sequenceid=276, filesize=12.0 K 2024-11-17T20:24:07,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ff04589cbd7acbe489756dc73d4cf811 in 1304ms, sequenceid=276, compaction requested=true 2024-11-17T20:24:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:07,247 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:07,247 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:07,250 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51706 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:07,251 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:07,251 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:07,251 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:07,251 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:07,251 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:07,251 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/5f921df19c014eb0bdacfb8cc670bc36, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/89cc054fa09c4335b89ce7b46c2492ab, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9979bd74fedf4b8d8fadbbb7a60158f0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/28c80648266c4c54b7cf52e49d234563] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=50.5 K 2024-11-17T20:24:07,251 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/bd11612c6b37442887425d849f70bb4c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/69df0b2fd95d4f0cba2afebae5bcf93a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/794a035b909544ffba1cf3921bfe8412, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3f0086b74c8e4092a8a9cb2b25079b44] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=48.1 K 2024-11-17T20:24:07,252 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bd11612c6b37442887425d849f70bb4c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731875041579 2024-11-17T20:24:07,252 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f921df19c014eb0bdacfb8cc670bc36, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731875041579 2024-11-17T20:24:07,253 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 69df0b2fd95d4f0cba2afebae5bcf93a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731875041941 2024-11-17T20:24:07,253 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cc054fa09c4335b89ce7b46c2492ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731875041941 2024-11-17T20:24:07,254 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9979bd74fedf4b8d8fadbbb7a60158f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731875042603 2024-11-17T20:24:07,254 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 794a035b909544ffba1cf3921bfe8412, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731875042603 2024-11-17T20:24:07,254 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c80648266c4c54b7cf52e49d234563, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731875044791 2024-11-17T20:24:07,255 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0086b74c8e4092a8a9cb2b25079b44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731875044791 2024-11-17T20:24:07,283 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:07,284 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/2ce8a2a57949459989d8598da4e9b68f is 50, key is test_row_0/A:col10/1731875044796/Put/seqid=0 2024-11-17T20:24:07,288 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:07,288 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/a827d853c6104f3ebd077abd32b80158 is 50, key is test_row_0/B:col10/1731875044796/Put/seqid=0 2024-11-17T20:24:07,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T20:24:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:07,298 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:24:07,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741895_1071 (size=12949) 2024-11-17T20:24:07,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/888761e2fa134ca89b8d7dc04cf71977 is 50, key is test_row_0/A:col10/1731875045952/Put/seqid=0 2024-11-17T20:24:07,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741894_1070 (size=12949) 2024-11-17T20:24:07,367 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/2ce8a2a57949459989d8598da4e9b68f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2ce8a2a57949459989d8598da4e9b68f 2024-11-17T20:24:07,377 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into 2ce8a2a57949459989d8598da4e9b68f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:07,377 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:07,377 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=12, startTime=1731875047246; duration=0sec 2024-11-17T20:24:07,378 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:07,378 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:07,378 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:07,379 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:07,379 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:07,379 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:07,380 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3725108eb528419e89ea56546edcedae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/261b286680ba453082a73ca475560d69, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/1cc8a14679ac41c4b0c4f3ab36a1444d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/f51850142a0e42e59c683b04c37b5cdf] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=48.1 K 2024-11-17T20:24:07,380 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3725108eb528419e89ea56546edcedae, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731875041579 2024-11-17T20:24:07,382 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261b286680ba453082a73ca475560d69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731875041941 2024-11-17T20:24:07,383 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cc8a14679ac41c4b0c4f3ab36a1444d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731875042603 2024-11-17T20:24:07,384 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f51850142a0e42e59c683b04c37b5cdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731875044791 2024-11-17T20:24:07,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741896_1072 (size=12301) 2024-11-17T20:24:07,388 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/888761e2fa134ca89b8d7dc04cf71977 2024-11-17T20:24:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/7fd9bd326bc64e9db91901dd21f95223 is 50, key is test_row_0/B:col10/1731875045952/Put/seqid=0 2024-11-17T20:24:07,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741897_1073 (size=12301) 2024-11-17T20:24:07,427 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/7fd9bd326bc64e9db91901dd21f95223 2024-11-17T20:24:07,427 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:07,428 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/ac7ba4ea0ddd423aa5f75202738aa0be is 50, key is test_row_0/C:col10/1731875044796/Put/seqid=0 2024-11-17T20:24:07,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741898_1074 (size=12949) 2024-11-17T20:24:07,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/b9d370fd4c164f6c8edc6c5af94e193f is 50, key is test_row_0/C:col10/1731875045952/Put/seqid=0 2024-11-17T20:24:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T20:24:07,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741899_1075 (size=12301) 2024-11-17T20:24:07,759 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/a827d853c6104f3ebd077abd32b80158 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a827d853c6104f3ebd077abd32b80158 2024-11-17T20:24:07,771 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into a827d853c6104f3ebd077abd32b80158(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:07,771 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:07,771 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=12, startTime=1731875047247; duration=0sec 2024-11-17T20:24:07,771 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:07,771 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:07,882 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/ac7ba4ea0ddd423aa5f75202738aa0be as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/ac7ba4ea0ddd423aa5f75202738aa0be 2024-11-17T20:24:07,901 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into ac7ba4ea0ddd423aa5f75202738aa0be(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:07,901 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:07,901 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=12, startTime=1731875047247; duration=0sec 2024-11-17T20:24:07,901 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:07,901 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:07,922 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/b9d370fd4c164f6c8edc6c5af94e193f 2024-11-17T20:24:07,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/888761e2fa134ca89b8d7dc04cf71977 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/888761e2fa134ca89b8d7dc04cf71977 2024-11-17T20:24:07,942 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/888761e2fa134ca89b8d7dc04cf71977, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T20:24:07,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/7fd9bd326bc64e9db91901dd21f95223 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/7fd9bd326bc64e9db91901dd21f95223 2024-11-17T20:24:07,956 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/7fd9bd326bc64e9db91901dd21f95223, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T20:24:07,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/b9d370fd4c164f6c8edc6c5af94e193f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/b9d370fd4c164f6c8edc6c5af94e193f 2024-11-17T20:24:07,969 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/b9d370fd4c164f6c8edc6c5af94e193f, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T20:24:07,971 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for ff04589cbd7acbe489756dc73d4cf811 in 673ms, sequenceid=291, compaction requested=false 2024-11-17T20:24:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T20:24:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T20:24:07,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T20:24:07,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-17T20:24:07,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6200 sec 2024-11-17T20:24:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:08,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:24:08,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/2cdf1436831a476d990c818517995392 is 50, key is test_row_0/A:col10/1731875048134/Put/seqid=0 2024-11-17T20:24:08,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741900_1076 (size=17181) 2024-11-17T20:24:08,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/2cdf1436831a476d990c818517995392 2024-11-17T20:24:08,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875108167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875108167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875108168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875108170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875108180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/78c50c19a9b34448ba7b536b31691d42 is 50, key is test_row_0/B:col10/1731875048134/Put/seqid=0 2024-11-17T20:24:08,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741901_1077 (size=12301) 2024-11-17T20:24:08,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875108280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875108280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875108289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875108289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875108290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875108489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875108489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875108499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875108500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875108500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T20:24:08,509 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:08,510 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T20:24:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T20:24:08,512 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:08,514 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:08,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:08,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/78c50c19a9b34448ba7b536b31691d42 2024-11-17T20:24:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T20:24:08,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/864cc666f32045c2bf574fed595020e5 is 50, key is test_row_0/C:col10/1731875048134/Put/seqid=0 2024-11-17T20:24:08,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741902_1078 (size=12301) 2024-11-17T20:24:08,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T20:24:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:08,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875108799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875108799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875108809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875108810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875108810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:08,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T20:24:08,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T20:24:08,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:08,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:08,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:08,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T20:24:08,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:08,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:08,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:08,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:09,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/864cc666f32045c2bf574fed595020e5 2024-11-17T20:24:09,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/2cdf1436831a476d990c818517995392 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2cdf1436831a476d990c818517995392 2024-11-17T20:24:09,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2cdf1436831a476d990c818517995392, entries=250, sequenceid=305, filesize=16.8 K 2024-11-17T20:24:09,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/78c50c19a9b34448ba7b536b31691d42 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/78c50c19a9b34448ba7b536b31691d42 2024-11-17T20:24:09,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/78c50c19a9b34448ba7b536b31691d42, entries=150, sequenceid=305, filesize=12.0 K 2024-11-17T20:24:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/864cc666f32045c2bf574fed595020e5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/864cc666f32045c2bf574fed595020e5 2024-11-17T20:24:09,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/864cc666f32045c2bf574fed595020e5, entries=150, sequenceid=305, filesize=12.0 K 2024-11-17T20:24:09,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ff04589cbd7acbe489756dc73d4cf811 in 925ms, sequenceid=305, compaction requested=true 2024-11-17T20:24:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:09,065 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:09,065 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:09,066 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:09,066 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:09,066 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:09,067 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2ce8a2a57949459989d8598da4e9b68f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/888761e2fa134ca89b8d7dc04cf71977, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2cdf1436831a476d990c818517995392] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=41.4 K 2024-11-17T20:24:09,067 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:09,067 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:09,067 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:09,067 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a827d853c6104f3ebd077abd32b80158, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/7fd9bd326bc64e9db91901dd21f95223, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/78c50c19a9b34448ba7b536b31691d42] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.7 K 2024-11-17T20:24:09,068 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ce8a2a57949459989d8598da4e9b68f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731875044791 2024-11-17T20:24:09,068 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a827d853c6104f3ebd077abd32b80158, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731875044791 2024-11-17T20:24:09,069 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd9bd326bc64e9db91901dd21f95223, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731875045952 2024-11-17T20:24:09,069 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888761e2fa134ca89b8d7dc04cf71977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731875045952 2024-11-17T20:24:09,070 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cdf1436831a476d990c818517995392, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731875048122 2024-11-17T20:24:09,070 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c50c19a9b34448ba7b536b31691d42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731875048125 2024-11-17T20:24:09,082 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#66 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:09,082 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/1277628d5e494e8daf630a6f2ada7f53 is 50, key is test_row_0/B:col10/1731875048134/Put/seqid=0 2024-11-17T20:24:09,082 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:09,083 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/094cd04e47e241a38cb591bee3457586 is 50, key is test_row_0/A:col10/1731875048134/Put/seqid=0 2024-11-17T20:24:09,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741903_1079 (size=13051) 2024-11-17T20:24:09,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741904_1080 (size=13051) 2024-11-17T20:24:09,113 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/094cd04e47e241a38cb591bee3457586 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/094cd04e47e241a38cb591bee3457586 2024-11-17T20:24:09,119 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into 094cd04e47e241a38cb591bee3457586(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:09,119 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:09,120 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875049065; duration=0sec 2024-11-17T20:24:09,120 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:09,120 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:09,120 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:09,121 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:09,121 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:09,121 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:09,121 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/ac7ba4ea0ddd423aa5f75202738aa0be, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/b9d370fd4c164f6c8edc6c5af94e193f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/864cc666f32045c2bf574fed595020e5] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.7 K 2024-11-17T20:24:09,122 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac7ba4ea0ddd423aa5f75202738aa0be, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731875044791 2024-11-17T20:24:09,123 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9d370fd4c164f6c8edc6c5af94e193f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731875045952 2024-11-17T20:24:09,123 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864cc666f32045c2bf574fed595020e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731875048125 2024-11-17T20:24:09,131 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:09,132 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/bd76fae7848645f9a815d9a437d0d238 is 50, key is test_row_0/C:col10/1731875048134/Put/seqid=0 2024-11-17T20:24:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T20:24:09,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T20:24:09,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:09,147 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:24:09,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741905_1081 (size=13051) 2024-11-17T20:24:09,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/454c22c6387b4e2e8217a182fdb2ffe8 is 50, key is test_row_0/A:col10/1731875048166/Put/seqid=0 2024-11-17T20:24:09,158 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/bd76fae7848645f9a815d9a437d0d238 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/bd76fae7848645f9a815d9a437d0d238 2024-11-17T20:24:09,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741906_1082 (size=12301) 2024-11-17T20:24:09,159 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/454c22c6387b4e2e8217a182fdb2ffe8 2024-11-17T20:24:09,165 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into bd76fae7848645f9a815d9a437d0d238(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:09,166 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:09,166 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875049065; duration=0sec 2024-11-17T20:24:09,166 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:09,166 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:09,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/eee2e8d671f448d4b768cdb8842d0382 is 50, key is test_row_0/B:col10/1731875048166/Put/seqid=0 2024-11-17T20:24:09,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741907_1083 (size=12301) 2024-11-17T20:24:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:09,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:09,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875109320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875109321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875109324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875109325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875109325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875109429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875109429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875109430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875109440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875109440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,535 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/1277628d5e494e8daf630a6f2ada7f53 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/1277628d5e494e8daf630a6f2ada7f53 2024-11-17T20:24:09,543 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into 1277628d5e494e8daf630a6f2ada7f53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:09,543 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:09,543 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875049065; duration=0sec 2024-11-17T20:24:09,543 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:09,543 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:09,574 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/eee2e8d671f448d4b768cdb8842d0382 2024-11-17T20:24:09,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/8289b894aba0408ba73c14fb03b8c40d is 50, key is test_row_0/C:col10/1731875048166/Put/seqid=0 2024-11-17T20:24:09,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741908_1084 (size=12301) 2024-11-17T20:24:09,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875109640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875109640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875109640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T20:24:09,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875109649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875109650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875109949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875109949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875109949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875109960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875109960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:09,995 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/8289b894aba0408ba73c14fb03b8c40d 2024-11-17T20:24:10,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/454c22c6387b4e2e8217a182fdb2ffe8 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/454c22c6387b4e2e8217a182fdb2ffe8 2024-11-17T20:24:10,007 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/454c22c6387b4e2e8217a182fdb2ffe8, entries=150, sequenceid=331, filesize=12.0 K 2024-11-17T20:24:10,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/eee2e8d671f448d4b768cdb8842d0382 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/eee2e8d671f448d4b768cdb8842d0382 2024-11-17T20:24:10,018 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/eee2e8d671f448d4b768cdb8842d0382, entries=150, sequenceid=331, filesize=12.0 K 2024-11-17T20:24:10,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/8289b894aba0408ba73c14fb03b8c40d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8289b894aba0408ba73c14fb03b8c40d 2024-11-17T20:24:10,026 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8289b894aba0408ba73c14fb03b8c40d, entries=150, sequenceid=331, filesize=12.0 K 2024-11-17T20:24:10,028 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for ff04589cbd7acbe489756dc73d4cf811 in 881ms, sequenceid=331, compaction requested=false 2024-11-17T20:24:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:10,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T20:24:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T20:24:10,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T20:24:10,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-11-17T20:24:10,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5250 sec 2024-11-17T20:24:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:10,462 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:10,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:24:10,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/76981cc70a964a1385b3ac78578b46e1 is 50, key is test_row_0/A:col10/1731875049323/Put/seqid=0 2024-11-17T20:24:10,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741909_1085 (size=12301) 2024-11-17T20:24:10,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875110486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875110489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875110490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875110491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875110494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875110600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875110600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875110600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875110601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875110610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T20:24:10,660 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:10,660 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T20:24:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T20:24:10,662 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:10,663 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:10,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T20:24:10,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875110810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875110810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875110810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875110810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T20:24:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:10,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875110819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:10,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/76981cc70a964a1385b3ac78578b46e1 2024-11-17T20:24:10,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/5c07d61d833848199112076216e9c1c4 is 50, key is test_row_0/B:col10/1731875049323/Put/seqid=0 2024-11-17T20:24:10,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741910_1086 (size=12301) 2024-11-17T20:24:10,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T20:24:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:10,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T20:24:11,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875111119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875111120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875111120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875111120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T20:24:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:11,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875111129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T20:24:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T20:24:11,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/5c07d61d833848199112076216e9c1c4 2024-11-17T20:24:11,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/757332ebb2ab4f75b225d9ad57d1d3cd is 50, key is test_row_0/C:col10/1731875049323/Put/seqid=0 2024-11-17T20:24:11,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741911_1087 (size=12301) 2024-11-17T20:24:11,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/757332ebb2ab4f75b225d9ad57d1d3cd 2024-11-17T20:24:11,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/76981cc70a964a1385b3ac78578b46e1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/76981cc70a964a1385b3ac78578b46e1 2024-11-17T20:24:11,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/76981cc70a964a1385b3ac78578b46e1, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T20:24:11,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/5c07d61d833848199112076216e9c1c4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/5c07d61d833848199112076216e9c1c4 2024-11-17T20:24:11,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/5c07d61d833848199112076216e9c1c4, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T20:24:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/757332ebb2ab4f75b225d9ad57d1d3cd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/757332ebb2ab4f75b225d9ad57d1d3cd 2024-11-17T20:24:11,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/757332ebb2ab4f75b225d9ad57d1d3cd, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T20:24:11,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for ff04589cbd7acbe489756dc73d4cf811 in 897ms, sequenceid=348, compaction requested=true 2024-11-17T20:24:11,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:11,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:11,359 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:11,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:11,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:11,359 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:11,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:11,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:11,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:11,360 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:11,360 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:11,361 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:11,361 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:11,361 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,361 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,361 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/094cd04e47e241a38cb591bee3457586, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/454c22c6387b4e2e8217a182fdb2ffe8, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/76981cc70a964a1385b3ac78578b46e1] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.8 K 2024-11-17T20:24:11,361 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/1277628d5e494e8daf630a6f2ada7f53, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/eee2e8d671f448d4b768cdb8842d0382, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/5c07d61d833848199112076216e9c1c4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.8 K 2024-11-17T20:24:11,361 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094cd04e47e241a38cb591bee3457586, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731875048125 2024-11-17T20:24:11,361 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1277628d5e494e8daf630a6f2ada7f53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731875048125 2024-11-17T20:24:11,361 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454c22c6387b4e2e8217a182fdb2ffe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731875048158 2024-11-17T20:24:11,361 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting eee2e8d671f448d4b768cdb8842d0382, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731875048158 2024-11-17T20:24:11,362 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76981cc70a964a1385b3ac78578b46e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731875049323 2024-11-17T20:24:11,362 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c07d61d833848199112076216e9c1c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731875049323 2024-11-17T20:24:11,374 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:11,374 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:11,375 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/b53e94e0f4c447e79595063e19a5adee is 50, key is test_row_0/B:col10/1731875049323/Put/seqid=0 2024-11-17T20:24:11,375 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/d065aaaca82e4a3796ceabf09dbf9274 is 50, key is test_row_0/A:col10/1731875049323/Put/seqid=0 2024-11-17T20:24:11,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741912_1088 (size=13153) 2024-11-17T20:24:11,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741913_1089 (size=13153) 2024-11-17T20:24:11,406 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/b53e94e0f4c447e79595063e19a5adee as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b53e94e0f4c447e79595063e19a5adee 2024-11-17T20:24:11,413 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into b53e94e0f4c447e79595063e19a5adee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:11,413 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:11,413 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875051359; duration=0sec 2024-11-17T20:24:11,413 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:11,413 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:11,413 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:11,416 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:11,417 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:11,417 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,417 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/bd76fae7848645f9a815d9a437d0d238, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8289b894aba0408ba73c14fb03b8c40d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/757332ebb2ab4f75b225d9ad57d1d3cd] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.8 K 2024-11-17T20:24:11,418 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bd76fae7848645f9a815d9a437d0d238, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731875048125 2024-11-17T20:24:11,418 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8289b894aba0408ba73c14fb03b8c40d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731875048158 2024-11-17T20:24:11,419 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 757332ebb2ab4f75b225d9ad57d1d3cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731875049323 2024-11-17T20:24:11,429 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:11,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T20:24:11,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:11,429 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-17T20:24:11,429 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5717482a3b264eeeabfe57a7e6b45e20 is 50, key is test_row_0/C:col10/1731875049323/Put/seqid=0 2024-11-17T20:24:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/afe48f710ebd4ec6a9eaee7b71b1e773 is 50, key is test_row_0/A:col10/1731875050490/Put/seqid=0 2024-11-17T20:24:11,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741914_1090 (size=13153) 2024-11-17T20:24:11,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741915_1091 (size=12301) 2024-11-17T20:24:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875111638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875111639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875111639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875111642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875111643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875111749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875111749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875111750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875111750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T20:24:11,808 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/d065aaaca82e4a3796ceabf09dbf9274 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d065aaaca82e4a3796ceabf09dbf9274 2024-11-17T20:24:11,819 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into d065aaaca82e4a3796ceabf09dbf9274(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:11,819 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:11,819 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875051359; duration=0sec 2024-11-17T20:24:11,819 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:11,819 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:11,842 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/afe48f710ebd4ec6a9eaee7b71b1e773 2024-11-17T20:24:11,851 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5717482a3b264eeeabfe57a7e6b45e20 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5717482a3b264eeeabfe57a7e6b45e20 2024-11-17T20:24:11,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/19bec6dfe2294b91bf05050f10b6c9da is 50, key is test_row_0/B:col10/1731875050490/Put/seqid=0 2024-11-17T20:24:11,866 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into 5717482a3b264eeeabfe57a7e6b45e20(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:11,866 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:11,866 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875051359; duration=0sec 2024-11-17T20:24:11,866 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:11,866 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:11,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741916_1092 (size=12301) 2024-11-17T20:24:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875111959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875111959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875111959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875111960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:11,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,270 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/19bec6dfe2294b91bf05050f10b6c9da 2024-11-17T20:24:12,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875112269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875112270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875112269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/e7fd6142c621459fa7f374f6e210f6bb is 50, key is test_row_0/C:col10/1731875050490/Put/seqid=0 2024-11-17T20:24:12,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875112280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741917_1093 (size=12301) 2024-11-17T20:24:12,293 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/e7fd6142c621459fa7f374f6e210f6bb 2024-11-17T20:24:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/afe48f710ebd4ec6a9eaee7b71b1e773 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/afe48f710ebd4ec6a9eaee7b71b1e773 2024-11-17T20:24:12,306 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/afe48f710ebd4ec6a9eaee7b71b1e773, entries=150, sequenceid=371, filesize=12.0 K 2024-11-17T20:24:12,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/19bec6dfe2294b91bf05050f10b6c9da as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/19bec6dfe2294b91bf05050f10b6c9da 2024-11-17T20:24:12,315 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/19bec6dfe2294b91bf05050f10b6c9da, entries=150, sequenceid=371, filesize=12.0 K 2024-11-17T20:24:12,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/e7fd6142c621459fa7f374f6e210f6bb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e7fd6142c621459fa7f374f6e210f6bb 2024-11-17T20:24:12,324 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e7fd6142c621459fa7f374f6e210f6bb, entries=150, sequenceid=371, filesize=12.0 K 2024-11-17T20:24:12,326 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ff04589cbd7acbe489756dc73d4cf811 in 897ms, sequenceid=371, compaction requested=false 2024-11-17T20:24:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T20:24:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T20:24:12,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T20:24:12,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6650 sec 2024-11-17T20:24:12,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6720 sec 2024-11-17T20:24:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:12,651 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:12,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:24:12,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/ea4529d9a96c4ee29ccc001ef37db57b is 50, key is test_row_0/A:col10/1731875051638/Put/seqid=0 2024-11-17T20:24:12,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741918_1094 (size=14741) 2024-11-17T20:24:12,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/ea4529d9a96c4ee29ccc001ef37db57b 2024-11-17T20:24:12,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/56013bd03e5d402e96dd970017f707df is 50, key is test_row_0/B:col10/1731875051638/Put/seqid=0 2024-11-17T20:24:12,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741919_1095 (size=12301) 2024-11-17T20:24:12,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/56013bd03e5d402e96dd970017f707df 2024-11-17T20:24:12,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3badf427ff6948c1bee69d08a2f2ca85 is 50, key is test_row_0/C:col10/1731875051638/Put/seqid=0 2024-11-17T20:24:12,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741920_1096 (size=12301) 2024-11-17T20:24:12,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875112732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875112780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875112780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875112780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875112789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T20:24:12,810 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:12,810 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-17T20:24:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T20:24:12,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:12,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:12,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:12,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875112839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T20:24:12,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T20:24:12,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:12,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:12,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:12,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:12,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875113049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T20:24:13,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:13,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:13,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:13,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:13,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:13,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3badf427ff6948c1bee69d08a2f2ca85 2024-11-17T20:24:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T20:24:13,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/ea4529d9a96c4ee29ccc001ef37db57b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea4529d9a96c4ee29ccc001ef37db57b 2024-11-17T20:24:13,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea4529d9a96c4ee29ccc001ef37db57b, entries=200, sequenceid=388, filesize=14.4 K 2024-11-17T20:24:13,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/56013bd03e5d402e96dd970017f707df as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/56013bd03e5d402e96dd970017f707df 2024-11-17T20:24:13,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/56013bd03e5d402e96dd970017f707df, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T20:24:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/3badf427ff6948c1bee69d08a2f2ca85 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3badf427ff6948c1bee69d08a2f2ca85 2024-11-17T20:24:13,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3badf427ff6948c1bee69d08a2f2ca85, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T20:24:13,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ff04589cbd7acbe489756dc73d4cf811 in 500ms, sequenceid=388, compaction requested=true 2024-11-17T20:24:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:13,152 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:13,152 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:13,153 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:13,153 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:13,153 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:13,153 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b53e94e0f4c447e79595063e19a5adee, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/19bec6dfe2294b91bf05050f10b6c9da, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/56013bd03e5d402e96dd970017f707df] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.9 K 2024-11-17T20:24:13,154 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:13,154 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:13,154 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:13,154 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d065aaaca82e4a3796ceabf09dbf9274, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/afe48f710ebd4ec6a9eaee7b71b1e773, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea4529d9a96c4ee29ccc001ef37db57b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=39.3 K 2024-11-17T20:24:13,154 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting b53e94e0f4c447e79595063e19a5adee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731875049323 2024-11-17T20:24:13,155 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d065aaaca82e4a3796ceabf09dbf9274, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731875049323 2024-11-17T20:24:13,155 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 19bec6dfe2294b91bf05050f10b6c9da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731875050482 2024-11-17T20:24:13,156 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe48f710ebd4ec6a9eaee7b71b1e773, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731875050482 2024-11-17T20:24:13,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 56013bd03e5d402e96dd970017f707df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731875051638 2024-11-17T20:24:13,157 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea4529d9a96c4ee29ccc001ef37db57b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731875051638 2024-11-17T20:24:13,174 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:13,175 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/be526da9eec749888046479b2f15d7f0 is 50, key is test_row_0/B:col10/1731875051638/Put/seqid=0 2024-11-17T20:24:13,176 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:13,177 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/8c95620181f9491494b9a7d386c6c5bf is 50, key is test_row_0/A:col10/1731875051638/Put/seqid=0 2024-11-17T20:24:13,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741921_1097 (size=13255) 2024-11-17T20:24:13,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741922_1098 (size=13255) 2024-11-17T20:24:13,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T20:24:13,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:13,273 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T20:24:13,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/dd03798d375c4ab993542082f239ae08 is 50, key is test_row_0/A:col10/1731875052718/Put/seqid=0 2024-11-17T20:24:13,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741923_1099 (size=12301) 2024-11-17T20:24:13,284 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/dd03798d375c4ab993542082f239ae08 2024-11-17T20:24:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/54f6b26e4e884402b9f7e078f739af7a is 50, key is test_row_0/B:col10/1731875052718/Put/seqid=0 2024-11-17T20:24:13,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741924_1100 (size=12301) 2024-11-17T20:24:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:13,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:13,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875113393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T20:24:13,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875113509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,596 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/8c95620181f9491494b9a7d386c6c5bf as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/8c95620181f9491494b9a7d386c6c5bf 2024-11-17T20:24:13,598 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/be526da9eec749888046479b2f15d7f0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/be526da9eec749888046479b2f15d7f0 2024-11-17T20:24:13,609 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into 8c95620181f9491494b9a7d386c6c5bf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:13,609 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:13,609 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875053152; duration=0sec 2024-11-17T20:24:13,609 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:13,609 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:13,609 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:13,611 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:13,611 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:13,611 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:13,611 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5717482a3b264eeeabfe57a7e6b45e20, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e7fd6142c621459fa7f374f6e210f6bb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3badf427ff6948c1bee69d08a2f2ca85] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=36.9 K 2024-11-17T20:24:13,612 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into be526da9eec749888046479b2f15d7f0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:13,612 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:13,612 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5717482a3b264eeeabfe57a7e6b45e20, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731875049323 2024-11-17T20:24:13,612 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875053152; duration=0sec 2024-11-17T20:24:13,613 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7fd6142c621459fa7f374f6e210f6bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731875050482 2024-11-17T20:24:13,614 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:13,614 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:13,614 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3badf427ff6948c1bee69d08a2f2ca85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731875051638 2024-11-17T20:24:13,625 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:13,626 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/4e6e8d67474d4e59b8569c3e960b5824 is 50, key is test_row_0/C:col10/1731875051638/Put/seqid=0 2024-11-17T20:24:13,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741925_1101 (size=13255) 2024-11-17T20:24:13,707 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/54f6b26e4e884402b9f7e078f739af7a 2024-11-17T20:24:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5147e44d4f424c0babf293d545b60b05 is 50, key is test_row_0/C:col10/1731875052718/Put/seqid=0 2024-11-17T20:24:13,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875113735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741926_1102 (size=12301) 2024-11-17T20:24:13,744 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5147e44d4f424c0babf293d545b60b05 2024-11-17T20:24:13,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/dd03798d375c4ab993542082f239ae08 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/dd03798d375c4ab993542082f239ae08 2024-11-17T20:24:13,756 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/dd03798d375c4ab993542082f239ae08, entries=150, sequenceid=409, filesize=12.0 K 2024-11-17T20:24:13,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/54f6b26e4e884402b9f7e078f739af7a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/54f6b26e4e884402b9f7e078f739af7a 2024-11-17T20:24:13,765 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/54f6b26e4e884402b9f7e078f739af7a, entries=150, sequenceid=409, filesize=12.0 K 2024-11-17T20:24:13,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/5147e44d4f424c0babf293d545b60b05 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5147e44d4f424c0babf293d545b60b05 2024-11-17T20:24:13,776 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5147e44d4f424c0babf293d545b60b05, entries=150, sequenceid=409, filesize=12.0 K 2024-11-17T20:24:13,779 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ff04589cbd7acbe489756dc73d4cf811 in 506ms, sequenceid=409, compaction requested=false 2024-11-17T20:24:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-17T20:24:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-17T20:24:13,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T20:24:13,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 969 msec 2024-11-17T20:24:13,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 977 msec 2024-11-17T20:24:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:13,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:13,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T20:24:13,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b8d584dbe8a146cf9ff712977a23cdec is 50, key is test_row_0/A:col10/1731875053386/Put/seqid=0 2024-11-17T20:24:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741927_1103 (size=12301) 2024-11-17T20:24:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875113824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875113829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875113829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875113831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875113940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875113940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875113940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875113940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T20:24:13,950 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:13,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-17T20:24:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T20:24:13,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:13,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:13,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875114039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,048 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/4e6e8d67474d4e59b8569c3e960b5824 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/4e6e8d67474d4e59b8569c3e960b5824 2024-11-17T20:24:14,054 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into 4e6e8d67474d4e59b8569c3e960b5824(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:14,054 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:14,054 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875053152; duration=0sec 2024-11-17T20:24:14,054 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:14,054 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T20:24:14,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T20:24:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:14,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875114149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875114150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875114150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875114150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b8d584dbe8a146cf9ff712977a23cdec 2024-11-17T20:24:14,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/0b21b494a2a04b9e94a5adb54a319d81 is 50, key is test_row_0/B:col10/1731875053386/Put/seqid=0 2024-11-17T20:24:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741928_1104 (size=12301) 2024-11-17T20:24:14,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T20:24:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T20:24:14,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T20:24:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875114459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875114460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875114460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875114460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54098 deadline: 1731875114550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,557 INFO [AsyncFSWAL-0-hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData-prefix:0b0b0ae8de03,32801,1731875028288 {}] wal.AbstractFSWAL(1368): Slow sync cost: 140 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38093,DS-96187b8f-658b-441b-a2a5-378169f6917f,DISK]] 2024-11-17T20:24:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T20:24:14,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/0b21b494a2a04b9e94a5adb54a319d81 2024-11-17T20:24:14,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/667773d5651146a59a5814ca0e69954e is 50, key is test_row_0/C:col10/1731875053386/Put/seqid=0 2024-11-17T20:24:14,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741929_1105 (size=12301) 2024-11-17T20:24:14,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T20:24:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T20:24:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:14,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:14,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54120 deadline: 1731875114969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54108 deadline: 1731875114969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54078 deadline: 1731875114969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54092 deadline: 1731875114969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:15,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T20:24:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. as already flushing 2024-11-17T20:24:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:15,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:15,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/667773d5651146a59a5814ca0e69954e 2024-11-17T20:24:15,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/b8d584dbe8a146cf9ff712977a23cdec as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8d584dbe8a146cf9ff712977a23cdec 2024-11-17T20:24:15,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8d584dbe8a146cf9ff712977a23cdec, entries=150, sequenceid=427, filesize=12.0 K 2024-11-17T20:24:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/0b21b494a2a04b9e94a5adb54a319d81 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/0b21b494a2a04b9e94a5adb54a319d81 2024-11-17T20:24:15,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/0b21b494a2a04b9e94a5adb54a319d81, entries=150, sequenceid=427, filesize=12.0 K 2024-11-17T20:24:15,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/667773d5651146a59a5814ca0e69954e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/667773d5651146a59a5814ca0e69954e 2024-11-17T20:24:15,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/667773d5651146a59a5814ca0e69954e, entries=150, sequenceid=427, filesize=12.0 K 2024-11-17T20:24:15,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ff04589cbd7acbe489756dc73d4cf811 in 1295ms, sequenceid=427, compaction requested=true 2024-11-17T20:24:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:15,088 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:15,088 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ff04589cbd7acbe489756dc73d4cf811:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:15,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:15,089 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:15,089 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:15,089 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/A is initiating minor compaction (all files) 2024-11-17T20:24:15,089 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/B is initiating minor compaction (all files) 2024-11-17T20:24:15,089 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/B in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T20:24:15,089 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/A in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:15,089 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/be526da9eec749888046479b2f15d7f0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/54f6b26e4e884402b9f7e078f739af7a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/0b21b494a2a04b9e94a5adb54a319d81] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=37.0 K 2024-11-17T20:24:15,090 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/8c95620181f9491494b9a7d386c6c5bf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/dd03798d375c4ab993542082f239ae08, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8d584dbe8a146cf9ff712977a23cdec] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=37.0 K 2024-11-17T20:24:15,090 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting be526da9eec749888046479b2f15d7f0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731875051638 2024-11-17T20:24:15,090 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c95620181f9491494b9a7d386c6c5bf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731875051638 2024-11-17T20:24:15,090 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f6b26e4e884402b9f7e078f739af7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731875052685 2024-11-17T20:24:15,090 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd03798d375c4ab993542082f239ae08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731875052685 2024-11-17T20:24:15,091 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b21b494a2a04b9e94a5adb54a319d81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731875053372 2024-11-17T20:24:15,091 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d584dbe8a146cf9ff712977a23cdec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731875053372 2024-11-17T20:24:15,100 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#B#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:15,100 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:15,100 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/c715468091fb4eacbe7e9c963dbf206e is 50, key is test_row_0/B:col10/1731875053386/Put/seqid=0 2024-11-17T20:24:15,101 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/d590d50d46c744bfafe12f8f2b2c9597 is 50, key is test_row_0/A:col10/1731875053386/Put/seqid=0 2024-11-17T20:24:15,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741930_1106 (size=13357) 2024-11-17T20:24:15,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741931_1107 (size=13357) 2024-11-17T20:24:15,117 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/d590d50d46c744bfafe12f8f2b2c9597 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d590d50d46c744bfafe12f8f2b2c9597 2024-11-17T20:24:15,130 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/A of ff04589cbd7acbe489756dc73d4cf811 into d590d50d46c744bfafe12f8f2b2c9597(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:15,130 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:15,130 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/A, priority=13, startTime=1731875055088; duration=0sec 2024-11-17T20:24:15,131 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:15,131 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:A 2024-11-17T20:24:15,131 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:15,133 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:15,133 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ff04589cbd7acbe489756dc73d4cf811/C is initiating minor compaction (all files) 2024-11-17T20:24:15,133 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ff04589cbd7acbe489756dc73d4cf811/C in TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:15,133 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/4e6e8d67474d4e59b8569c3e960b5824, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5147e44d4f424c0babf293d545b60b05, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/667773d5651146a59a5814ca0e69954e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp, totalSize=37.0 K 2024-11-17T20:24:15,134 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e6e8d67474d4e59b8569c3e960b5824, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731875051638 2024-11-17T20:24:15,134 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5147e44d4f424c0babf293d545b60b05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731875052685 2024-11-17T20:24:15,135 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 667773d5651146a59a5814ca0e69954e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731875053372 2024-11-17T20:24:15,150 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ff04589cbd7acbe489756dc73d4cf811#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:15,151 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/57a47db1511444bbbab9adbf0948fd29 is 50, key is test_row_0/C:col10/1731875053386/Put/seqid=0 2024-11-17T20:24:15,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741932_1108 (size=13357) 2024-11-17T20:24:15,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T20:24:15,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:15,167 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T20:24:15,172 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/57a47db1511444bbbab9adbf0948fd29 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/57a47db1511444bbbab9adbf0948fd29 2024-11-17T20:24:15,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/fb811b4b580b443886565199a5bfa020 is 50, key is test_row_0/A:col10/1731875053828/Put/seqid=0 2024-11-17T20:24:15,182 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/C of ff04589cbd7acbe489756dc73d4cf811 into 57a47db1511444bbbab9adbf0948fd29(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:15,182 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:15,182 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/C, priority=13, startTime=1731875055088; duration=0sec 2024-11-17T20:24:15,182 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:15,182 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:C 2024-11-17T20:24:15,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741933_1109 (size=12301) 2024-11-17T20:24:15,191 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/fb811b4b580b443886565199a5bfa020 2024-11-17T20:24:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/08549166a53145a695e76f644119ff7a is 50, key is test_row_0/B:col10/1731875053828/Put/seqid=0 2024-11-17T20:24:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741934_1110 (size=12301) 2024-11-17T20:24:15,211 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/08549166a53145a695e76f644119ff7a 2024-11-17T20:24:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/91f345dca67a4d1dbc886c45154e0ef6 is 50, key is test_row_0/C:col10/1731875053828/Put/seqid=0 2024-11-17T20:24:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741935_1111 (size=12301) 2024-11-17T20:24:15,256 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/91f345dca67a4d1dbc886c45154e0ef6 2024-11-17T20:24:15,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/fb811b4b580b443886565199a5bfa020 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/fb811b4b580b443886565199a5bfa020 2024-11-17T20:24:15,273 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/fb811b4b580b443886565199a5bfa020, entries=150, sequenceid=449, filesize=12.0 K 2024-11-17T20:24:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/08549166a53145a695e76f644119ff7a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/08549166a53145a695e76f644119ff7a 2024-11-17T20:24:15,280 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/08549166a53145a695e76f644119ff7a, entries=150, sequenceid=449, filesize=12.0 K 2024-11-17T20:24:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/91f345dca67a4d1dbc886c45154e0ef6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/91f345dca67a4d1dbc886c45154e0ef6 2024-11-17T20:24:15,291 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/91f345dca67a4d1dbc886c45154e0ef6, entries=150, sequenceid=449, filesize=12.0 K 2024-11-17T20:24:15,294 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ff04589cbd7acbe489756dc73d4cf811 in 126ms, sequenceid=449, compaction requested=false 2024-11-17T20:24:15,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:15,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:15,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-17T20:24:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-17T20:24:15,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T20:24:15,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3430 sec 2024-11-17T20:24:15,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.3510 sec 2024-11-17T20:24:15,430 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T20:24:15,430 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,431 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T20:24:15,431 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,435 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T20:24:15,435 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,436 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,436 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,436 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,436 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,436 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,437 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,439 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T20:24:15,439 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,439 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,439 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,512 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/c715468091fb4eacbe7e9c963dbf206e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/c715468091fb4eacbe7e9c963dbf206e 2024-11-17T20:24:15,517 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ff04589cbd7acbe489756dc73d4cf811/B of ff04589cbd7acbe489756dc73d4cf811 into c715468091fb4eacbe7e9c963dbf206e(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:15,517 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ff04589cbd7acbe489756dc73d4cf811: 2024-11-17T20:24:15,517 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., storeName=ff04589cbd7acbe489756dc73d4cf811/B, priority=13, startTime=1731875055088; duration=0sec 2024-11-17T20:24:15,517 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:15,517 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ff04589cbd7acbe489756dc73d4cf811:B 2024-11-17T20:24:15,563 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T20:24:15,563 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,563 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,564 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,983 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T20:24:15,983 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T20:24:15,983 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T20:24:15,983 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,983 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,983 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,983 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,983 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,983 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,983 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,983 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,983 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:15,991 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T20:24:15,991 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:15,991 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,991 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:15,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T20:24:16,100 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:16,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3616 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3587 2024-11-17T20:24:16,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T20:24:16,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1618 2024-11-17T20:24:16,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4853 rows 2024-11-17T20:24:16,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1609 2024-11-17T20:24:16,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4826 rows 2024-11-17T20:24:16,102 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:24:16,103 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:24:16,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:16,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:16,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:16,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T20:24:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T20:24:16,117 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875056117"}]},"ts":"1731875056117"} 2024-11-17T20:24:16,118 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T20:24:16,119 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T20:24:16,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:24:16,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, UNASSIGN}] 2024-11-17T20:24:16,125 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, UNASSIGN 2024-11-17T20:24:16,127 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ff04589cbd7acbe489756dc73d4cf811, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:16,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, UNASSIGN because future has completed 2024-11-17T20:24:16,129 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:24:16,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:24:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T20:24:16,290 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:16,291 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:24:16,293 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing ff04589cbd7acbe489756dc73d4cf811, disabling compactions & flushes 2024-11-17T20:24:16,293 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:16,293 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:16,293 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. after waiting 0 ms 2024-11-17T20:24:16,293 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:16,294 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing ff04589cbd7acbe489756dc73d4cf811 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T20:24:16,301 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/6115c8d848b542958b39ae11bf95c3ba is 50, key is test_row_0/A:col10/1731875055980/Put/seqid=0 2024-11-17T20:24:16,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741936_1112 (size=12301) 2024-11-17T20:24:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T20:24:16,706 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/6115c8d848b542958b39ae11bf95c3ba 2024-11-17T20:24:16,714 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/ab1b6dc794234f9da3a40fb1fffa4252 is 50, key is test_row_0/B:col10/1731875055980/Put/seqid=0 2024-11-17T20:24:16,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741937_1113 (size=12301) 2024-11-17T20:24:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T20:24:17,118 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/ab1b6dc794234f9da3a40fb1fffa4252 2024-11-17T20:24:17,130 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/a472abd7840f4440847a2785820b0c26 is 50, key is test_row_0/C:col10/1731875055980/Put/seqid=0 2024-11-17T20:24:17,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741938_1114 (size=12301) 2024-11-17T20:24:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T20:24:17,390 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T20:24:17,536 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/a472abd7840f4440847a2785820b0c26 2024-11-17T20:24:17,547 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/A/6115c8d848b542958b39ae11bf95c3ba as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/6115c8d848b542958b39ae11bf95c3ba 2024-11-17T20:24:17,552 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/6115c8d848b542958b39ae11bf95c3ba, entries=150, sequenceid=459, filesize=12.0 K 2024-11-17T20:24:17,554 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/B/ab1b6dc794234f9da3a40fb1fffa4252 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/ab1b6dc794234f9da3a40fb1fffa4252 2024-11-17T20:24:17,558 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/ab1b6dc794234f9da3a40fb1fffa4252, entries=150, sequenceid=459, filesize=12.0 K 2024-11-17T20:24:17,559 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/.tmp/C/a472abd7840f4440847a2785820b0c26 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a472abd7840f4440847a2785820b0c26 2024-11-17T20:24:17,564 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a472abd7840f4440847a2785820b0c26, entries=150, sequenceid=459, filesize=12.0 K 2024-11-17T20:24:17,565 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ff04589cbd7acbe489756dc73d4cf811 in 1272ms, sequenceid=459, compaction requested=true 2024-11-17T20:24:17,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/02d5e645eefc475bb494afba99a9f2df, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/f8fc460e25d34672a9d270ef04bb6485, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/c39ae7fc3ae3440bb17efe459c3b843d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/84d136af65c84a6ab8666471ec64df94, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b5a9e39a45484c858f03966f0758d6d1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/3941b17d7f7243e3bc1900deab3a3680, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b200b5314649442ca3ee5b9951866f26, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea8641cf42b1415ca5d3e98088f9897d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d2d3f57d25ea4576af47e658dd4c2ef0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/7f0bf5d5fd954f7da35c7d542664aeb1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9007d52809614281a90ead46ffacb61c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/12c7060653164c0aa24bc3598fd3ff15, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/a68eefd457144697b3cd8616a8005277, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/5f921df19c014eb0bdacfb8cc670bc36, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8fb03b636464cb78d18aa331c8a05b6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/89cc054fa09c4335b89ce7b46c2492ab, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9979bd74fedf4b8d8fadbbb7a60158f0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/28c80648266c4c54b7cf52e49d234563, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2ce8a2a57949459989d8598da4e9b68f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/888761e2fa134ca89b8d7dc04cf71977, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2cdf1436831a476d990c818517995392, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/094cd04e47e241a38cb591bee3457586, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/454c22c6387b4e2e8217a182fdb2ffe8, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d065aaaca82e4a3796ceabf09dbf9274, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/76981cc70a964a1385b3ac78578b46e1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/afe48f710ebd4ec6a9eaee7b71b1e773, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea4529d9a96c4ee29ccc001ef37db57b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/8c95620181f9491494b9a7d386c6c5bf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/dd03798d375c4ab993542082f239ae08, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8d584dbe8a146cf9ff712977a23cdec] to archive 2024-11-17T20:24:17,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:24:17,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/02d5e645eefc475bb494afba99a9f2df to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/02d5e645eefc475bb494afba99a9f2df 2024-11-17T20:24:17,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/f8fc460e25d34672a9d270ef04bb6485 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/f8fc460e25d34672a9d270ef04bb6485 2024-11-17T20:24:17,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/c39ae7fc3ae3440bb17efe459c3b843d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/c39ae7fc3ae3440bb17efe459c3b843d 2024-11-17T20:24:17,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/84d136af65c84a6ab8666471ec64df94 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/84d136af65c84a6ab8666471ec64df94 2024-11-17T20:24:17,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b5a9e39a45484c858f03966f0758d6d1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b5a9e39a45484c858f03966f0758d6d1 2024-11-17T20:24:17,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/3941b17d7f7243e3bc1900deab3a3680 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/3941b17d7f7243e3bc1900deab3a3680 2024-11-17T20:24:17,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b200b5314649442ca3ee5b9951866f26 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b200b5314649442ca3ee5b9951866f26 2024-11-17T20:24:17,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea8641cf42b1415ca5d3e98088f9897d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea8641cf42b1415ca5d3e98088f9897d 2024-11-17T20:24:17,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d2d3f57d25ea4576af47e658dd4c2ef0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d2d3f57d25ea4576af47e658dd4c2ef0 2024-11-17T20:24:17,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/bbd8a3ab77a54f0c961a8fb1cd5d35a2 2024-11-17T20:24:17,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/7f0bf5d5fd954f7da35c7d542664aeb1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/7f0bf5d5fd954f7da35c7d542664aeb1 2024-11-17T20:24:17,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9007d52809614281a90ead46ffacb61c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9007d52809614281a90ead46ffacb61c 2024-11-17T20:24:17,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/12c7060653164c0aa24bc3598fd3ff15 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/12c7060653164c0aa24bc3598fd3ff15 2024-11-17T20:24:17,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/a68eefd457144697b3cd8616a8005277 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/a68eefd457144697b3cd8616a8005277 2024-11-17T20:24:17,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/5f921df19c014eb0bdacfb8cc670bc36 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/5f921df19c014eb0bdacfb8cc670bc36 2024-11-17T20:24:17,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8fb03b636464cb78d18aa331c8a05b6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8fb03b636464cb78d18aa331c8a05b6 2024-11-17T20:24:17,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/89cc054fa09c4335b89ce7b46c2492ab to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/89cc054fa09c4335b89ce7b46c2492ab 2024-11-17T20:24:17,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9979bd74fedf4b8d8fadbbb7a60158f0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/9979bd74fedf4b8d8fadbbb7a60158f0 2024-11-17T20:24:17,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/28c80648266c4c54b7cf52e49d234563 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/28c80648266c4c54b7cf52e49d234563 2024-11-17T20:24:17,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2ce8a2a57949459989d8598da4e9b68f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2ce8a2a57949459989d8598da4e9b68f 2024-11-17T20:24:17,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/888761e2fa134ca89b8d7dc04cf71977 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/888761e2fa134ca89b8d7dc04cf71977 2024-11-17T20:24:17,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2cdf1436831a476d990c818517995392 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/2cdf1436831a476d990c818517995392 2024-11-17T20:24:17,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/094cd04e47e241a38cb591bee3457586 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/094cd04e47e241a38cb591bee3457586 2024-11-17T20:24:17,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/454c22c6387b4e2e8217a182fdb2ffe8 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/454c22c6387b4e2e8217a182fdb2ffe8 2024-11-17T20:24:17,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d065aaaca82e4a3796ceabf09dbf9274 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d065aaaca82e4a3796ceabf09dbf9274 2024-11-17T20:24:17,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/76981cc70a964a1385b3ac78578b46e1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/76981cc70a964a1385b3ac78578b46e1 2024-11-17T20:24:17,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/afe48f710ebd4ec6a9eaee7b71b1e773 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/afe48f710ebd4ec6a9eaee7b71b1e773 2024-11-17T20:24:17,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea4529d9a96c4ee29ccc001ef37db57b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/ea4529d9a96c4ee29ccc001ef37db57b 2024-11-17T20:24:17,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/8c95620181f9491494b9a7d386c6c5bf to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/8c95620181f9491494b9a7d386c6c5bf 2024-11-17T20:24:17,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/dd03798d375c4ab993542082f239ae08 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/dd03798d375c4ab993542082f239ae08 2024-11-17T20:24:17,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8d584dbe8a146cf9ff712977a23cdec to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/b8d584dbe8a146cf9ff712977a23cdec 2024-11-17T20:24:17,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/f5b59a587462436e8b8922fffc56071d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/486697311638400f91a8be8b0d0e10bb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/8d119145a8a949498c600f413a296cd5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/af5f1892c5c64139b798a3f972b6b91b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/fbeaf05e07474eecb3fd1c675cff5280, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/9fbaadbb2f254062838a0fba90428f02, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a5c96aa53b1c4ec0a4aaf173a994b7b4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/cb8afd750d4d4d1da713e90297d104a6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/118103c182fa460a8db46a9a16db7b65, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/43ecf894cd10411c9475cb6d8dcee802, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/52dcf74af15e4984881aab6a5bc5a96b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/4b70c9e722e342948c7f0ba77bc9cb72, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3d3ad21d1a004b29b96db234f9b214cf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b14c99a733444d0ca12a3a5891d6f0dd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/bd11612c6b37442887425d849f70bb4c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3849d4b96060442a9a158da875ca8d87, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/69df0b2fd95d4f0cba2afebae5bcf93a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/794a035b909544ffba1cf3921bfe8412, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a827d853c6104f3ebd077abd32b80158, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3f0086b74c8e4092a8a9cb2b25079b44, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/7fd9bd326bc64e9db91901dd21f95223, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/1277628d5e494e8daf630a6f2ada7f53, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/78c50c19a9b34448ba7b536b31691d42, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/eee2e8d671f448d4b768cdb8842d0382, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b53e94e0f4c447e79595063e19a5adee, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/5c07d61d833848199112076216e9c1c4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/19bec6dfe2294b91bf05050f10b6c9da, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/be526da9eec749888046479b2f15d7f0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/56013bd03e5d402e96dd970017f707df, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/54f6b26e4e884402b9f7e078f739af7a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/0b21b494a2a04b9e94a5adb54a319d81] to archive 2024-11-17T20:24:17,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:24:17,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/f5b59a587462436e8b8922fffc56071d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/f5b59a587462436e8b8922fffc56071d 2024-11-17T20:24:17,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/486697311638400f91a8be8b0d0e10bb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/486697311638400f91a8be8b0d0e10bb 2024-11-17T20:24:17,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/8d119145a8a949498c600f413a296cd5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/8d119145a8a949498c600f413a296cd5 2024-11-17T20:24:17,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/af5f1892c5c64139b798a3f972b6b91b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/af5f1892c5c64139b798a3f972b6b91b 2024-11-17T20:24:17,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/fbeaf05e07474eecb3fd1c675cff5280 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/fbeaf05e07474eecb3fd1c675cff5280 2024-11-17T20:24:17,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/9fbaadbb2f254062838a0fba90428f02 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/9fbaadbb2f254062838a0fba90428f02 2024-11-17T20:24:17,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a5c96aa53b1c4ec0a4aaf173a994b7b4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a5c96aa53b1c4ec0a4aaf173a994b7b4 2024-11-17T20:24:17,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/cb8afd750d4d4d1da713e90297d104a6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/cb8afd750d4d4d1da713e90297d104a6 2024-11-17T20:24:17,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/118103c182fa460a8db46a9a16db7b65 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/118103c182fa460a8db46a9a16db7b65 2024-11-17T20:24:17,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/43ecf894cd10411c9475cb6d8dcee802 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/43ecf894cd10411c9475cb6d8dcee802 2024-11-17T20:24:17,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/52dcf74af15e4984881aab6a5bc5a96b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/52dcf74af15e4984881aab6a5bc5a96b 2024-11-17T20:24:17,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/4b70c9e722e342948c7f0ba77bc9cb72 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/4b70c9e722e342948c7f0ba77bc9cb72 2024-11-17T20:24:17,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3d3ad21d1a004b29b96db234f9b214cf to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3d3ad21d1a004b29b96db234f9b214cf 2024-11-17T20:24:17,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b14c99a733444d0ca12a3a5891d6f0dd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b14c99a733444d0ca12a3a5891d6f0dd 2024-11-17T20:24:17,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/bd11612c6b37442887425d849f70bb4c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/bd11612c6b37442887425d849f70bb4c 2024-11-17T20:24:17,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3849d4b96060442a9a158da875ca8d87 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3849d4b96060442a9a158da875ca8d87 2024-11-17T20:24:17,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/69df0b2fd95d4f0cba2afebae5bcf93a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/69df0b2fd95d4f0cba2afebae5bcf93a 2024-11-17T20:24:17,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/794a035b909544ffba1cf3921bfe8412 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/794a035b909544ffba1cf3921bfe8412 2024-11-17T20:24:17,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a827d853c6104f3ebd077abd32b80158 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/a827d853c6104f3ebd077abd32b80158 2024-11-17T20:24:17,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3f0086b74c8e4092a8a9cb2b25079b44 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/3f0086b74c8e4092a8a9cb2b25079b44 2024-11-17T20:24:17,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/7fd9bd326bc64e9db91901dd21f95223 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/7fd9bd326bc64e9db91901dd21f95223 2024-11-17T20:24:17,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/1277628d5e494e8daf630a6f2ada7f53 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/1277628d5e494e8daf630a6f2ada7f53 2024-11-17T20:24:17,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/78c50c19a9b34448ba7b536b31691d42 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/78c50c19a9b34448ba7b536b31691d42 2024-11-17T20:24:17,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/eee2e8d671f448d4b768cdb8842d0382 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/eee2e8d671f448d4b768cdb8842d0382 2024-11-17T20:24:17,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b53e94e0f4c447e79595063e19a5adee to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/b53e94e0f4c447e79595063e19a5adee 2024-11-17T20:24:17,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/5c07d61d833848199112076216e9c1c4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/5c07d61d833848199112076216e9c1c4 2024-11-17T20:24:17,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/19bec6dfe2294b91bf05050f10b6c9da to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/19bec6dfe2294b91bf05050f10b6c9da 2024-11-17T20:24:17,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/be526da9eec749888046479b2f15d7f0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/be526da9eec749888046479b2f15d7f0 2024-11-17T20:24:17,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/56013bd03e5d402e96dd970017f707df to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/56013bd03e5d402e96dd970017f707df 2024-11-17T20:24:17,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/54f6b26e4e884402b9f7e078f739af7a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/54f6b26e4e884402b9f7e078f739af7a 2024-11-17T20:24:17,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/0b21b494a2a04b9e94a5adb54a319d81 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/0b21b494a2a04b9e94a5adb54a319d81 2024-11-17T20:24:17,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3349e978ff0f4a028e6733a11a68472e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8bd00c1e40e6469595c5723eeb8441eb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31485b7e597f446a8faacb0de1470d5b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31223b12639d4368b06a2f7863320e24, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5802a35baf4a40b0ad3888d8c87f39ec, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9f965244273c4f27a14fe50d9839e4a2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/aa692720d180433e86d63bb73104f8b2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9765d18fc72b4f53a3d952738ad80741, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/cfb57867118244c0ab2607661e7257cb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5d75a8a6eb2a43a09640adf4c795731e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e31eea5fd0d3444ea59d5361a07a4dd4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/d92d42405378412d8ee87395b07f89d1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3e213d1b1fad466f9cdd1937641467d2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/6a851775750741c9be822ac326e4ca00, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3725108eb528419e89ea56546edcedae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a8430b5086644a90bfa1ec11ab8a4298, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/261b286680ba453082a73ca475560d69, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/1cc8a14679ac41c4b0c4f3ab36a1444d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/ac7ba4ea0ddd423aa5f75202738aa0be, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/f51850142a0e42e59c683b04c37b5cdf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/b9d370fd4c164f6c8edc6c5af94e193f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/bd76fae7848645f9a815d9a437d0d238, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/864cc666f32045c2bf574fed595020e5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8289b894aba0408ba73c14fb03b8c40d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5717482a3b264eeeabfe57a7e6b45e20, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/757332ebb2ab4f75b225d9ad57d1d3cd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e7fd6142c621459fa7f374f6e210f6bb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/4e6e8d67474d4e59b8569c3e960b5824, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3badf427ff6948c1bee69d08a2f2ca85, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5147e44d4f424c0babf293d545b60b05, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/667773d5651146a59a5814ca0e69954e] to archive 2024-11-17T20:24:17,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:24:17,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3349e978ff0f4a028e6733a11a68472e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3349e978ff0f4a028e6733a11a68472e 2024-11-17T20:24:17,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8bd00c1e40e6469595c5723eeb8441eb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8bd00c1e40e6469595c5723eeb8441eb 2024-11-17T20:24:17,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31485b7e597f446a8faacb0de1470d5b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31485b7e597f446a8faacb0de1470d5b 2024-11-17T20:24:17,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31223b12639d4368b06a2f7863320e24 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/31223b12639d4368b06a2f7863320e24 2024-11-17T20:24:17,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5802a35baf4a40b0ad3888d8c87f39ec to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5802a35baf4a40b0ad3888d8c87f39ec 2024-11-17T20:24:17,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9f965244273c4f27a14fe50d9839e4a2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9f965244273c4f27a14fe50d9839e4a2 2024-11-17T20:24:17,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/aa692720d180433e86d63bb73104f8b2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/aa692720d180433e86d63bb73104f8b2 2024-11-17T20:24:17,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9765d18fc72b4f53a3d952738ad80741 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/9765d18fc72b4f53a3d952738ad80741 2024-11-17T20:24:17,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/cfb57867118244c0ab2607661e7257cb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/cfb57867118244c0ab2607661e7257cb 2024-11-17T20:24:17,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5d75a8a6eb2a43a09640adf4c795731e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5d75a8a6eb2a43a09640adf4c795731e 2024-11-17T20:24:17,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e31eea5fd0d3444ea59d5361a07a4dd4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e31eea5fd0d3444ea59d5361a07a4dd4 2024-11-17T20:24:17,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/d92d42405378412d8ee87395b07f89d1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/d92d42405378412d8ee87395b07f89d1 2024-11-17T20:24:17,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3e213d1b1fad466f9cdd1937641467d2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3e213d1b1fad466f9cdd1937641467d2 2024-11-17T20:24:17,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/6a851775750741c9be822ac326e4ca00 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/6a851775750741c9be822ac326e4ca00 2024-11-17T20:24:17,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3725108eb528419e89ea56546edcedae to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3725108eb528419e89ea56546edcedae 2024-11-17T20:24:17,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a8430b5086644a90bfa1ec11ab8a4298 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a8430b5086644a90bfa1ec11ab8a4298 2024-11-17T20:24:17,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/261b286680ba453082a73ca475560d69 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/261b286680ba453082a73ca475560d69 2024-11-17T20:24:17,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/1cc8a14679ac41c4b0c4f3ab36a1444d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/1cc8a14679ac41c4b0c4f3ab36a1444d 2024-11-17T20:24:17,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/ac7ba4ea0ddd423aa5f75202738aa0be to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/ac7ba4ea0ddd423aa5f75202738aa0be 2024-11-17T20:24:17,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/f51850142a0e42e59c683b04c37b5cdf to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/f51850142a0e42e59c683b04c37b5cdf 2024-11-17T20:24:17,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/b9d370fd4c164f6c8edc6c5af94e193f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/b9d370fd4c164f6c8edc6c5af94e193f 2024-11-17T20:24:17,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/bd76fae7848645f9a815d9a437d0d238 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/bd76fae7848645f9a815d9a437d0d238 2024-11-17T20:24:17,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/864cc666f32045c2bf574fed595020e5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/864cc666f32045c2bf574fed595020e5 2024-11-17T20:24:17,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8289b894aba0408ba73c14fb03b8c40d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/8289b894aba0408ba73c14fb03b8c40d 2024-11-17T20:24:17,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5717482a3b264eeeabfe57a7e6b45e20 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5717482a3b264eeeabfe57a7e6b45e20 2024-11-17T20:24:17,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/757332ebb2ab4f75b225d9ad57d1d3cd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/757332ebb2ab4f75b225d9ad57d1d3cd 2024-11-17T20:24:17,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e7fd6142c621459fa7f374f6e210f6bb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/e7fd6142c621459fa7f374f6e210f6bb 2024-11-17T20:24:17,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/4e6e8d67474d4e59b8569c3e960b5824 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/4e6e8d67474d4e59b8569c3e960b5824 2024-11-17T20:24:17,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3badf427ff6948c1bee69d08a2f2ca85 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/3badf427ff6948c1bee69d08a2f2ca85 2024-11-17T20:24:17,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5147e44d4f424c0babf293d545b60b05 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/5147e44d4f424c0babf293d545b60b05 2024-11-17T20:24:17,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/667773d5651146a59a5814ca0e69954e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/667773d5651146a59a5814ca0e69954e 2024-11-17T20:24:17,693 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=1 2024-11-17T20:24:17,694 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. 2024-11-17T20:24:17,694 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for ff04589cbd7acbe489756dc73d4cf811: Waiting for close lock at 1731875056292Running coprocessor pre-close hooks at 1731875056293 (+1 ms)Disabling compacts and flushes for region at 1731875056293Disabling writes for close at 1731875056293Obtaining lock to block concurrent updates at 1731875056294 (+1 ms)Preparing flush snapshotting stores in ff04589cbd7acbe489756dc73d4cf811 at 1731875056294Finished memstore snapshotting TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731875056295 (+1 ms)Flushing stores of TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811. at 1731875056296 (+1 ms)Flushing ff04589cbd7acbe489756dc73d4cf811/A: creating writer at 1731875056296Flushing ff04589cbd7acbe489756dc73d4cf811/A: appending metadata at 1731875056301 (+5 ms)Flushing ff04589cbd7acbe489756dc73d4cf811/A: closing flushed file at 1731875056301Flushing ff04589cbd7acbe489756dc73d4cf811/B: creating writer at 1731875056711 (+410 ms)Flushing ff04589cbd7acbe489756dc73d4cf811/B: appending metadata at 1731875056713 (+2 ms)Flushing ff04589cbd7acbe489756dc73d4cf811/B: closing flushed file at 1731875056713Flushing ff04589cbd7acbe489756dc73d4cf811/C: creating writer at 1731875057127 (+414 ms)Flushing ff04589cbd7acbe489756dc73d4cf811/C: appending metadata at 1731875057130 (+3 ms)Flushing ff04589cbd7acbe489756dc73d4cf811/C: closing flushed file at 1731875057130Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ddaa859: reopening flushed file at 1731875057546 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b1d7ed4: reopening flushed file at 1731875057553 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2faa3ce1: reopening flushed file at 1731875057558 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ff04589cbd7acbe489756dc73d4cf811 in 1272ms, sequenceid=459, compaction requested=true at 1731875057565 (+7 ms)Writing region close event to WAL at 1731875057689 (+124 ms)Running coprocessor post-close hooks at 1731875057694 (+5 ms)Closed at 1731875057694 2024-11-17T20:24:17,696 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:17,697 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ff04589cbd7acbe489756dc73d4cf811, regionState=CLOSED 2024-11-17T20:24:17,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:24:17,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T20:24:17,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure ff04589cbd7acbe489756dc73d4cf811, server=0b0b0ae8de03,46555,1731875029029 in 1.5700 sec 2024-11-17T20:24:17,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-17T20:24:17,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ff04589cbd7acbe489756dc73d4cf811, UNASSIGN in 1.5770 sec 2024-11-17T20:24:17,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T20:24:17,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5840 sec 2024-11-17T20:24:17,707 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875057707"}]},"ts":"1731875057707"} 2024-11-17T20:24:17,708 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T20:24:17,708 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T20:24:17,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-11-17T20:24:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T20:24:18,270 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:18,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T20:24:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:18,285 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T20:24:18,287 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:18,291 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:18,295 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/recovered.edits] 2024-11-17T20:24:18,298 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/6115c8d848b542958b39ae11bf95c3ba to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/6115c8d848b542958b39ae11bf95c3ba 2024-11-17T20:24:18,299 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d590d50d46c744bfafe12f8f2b2c9597 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/d590d50d46c744bfafe12f8f2b2c9597 2024-11-17T20:24:18,300 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/fb811b4b580b443886565199a5bfa020 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/A/fb811b4b580b443886565199a5bfa020 2024-11-17T20:24:18,302 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/08549166a53145a695e76f644119ff7a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/08549166a53145a695e76f644119ff7a 2024-11-17T20:24:18,303 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/ab1b6dc794234f9da3a40fb1fffa4252 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/ab1b6dc794234f9da3a40fb1fffa4252 2024-11-17T20:24:18,304 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/c715468091fb4eacbe7e9c963dbf206e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/B/c715468091fb4eacbe7e9c963dbf206e 2024-11-17T20:24:18,306 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/57a47db1511444bbbab9adbf0948fd29 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/57a47db1511444bbbab9adbf0948fd29 2024-11-17T20:24:18,307 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/91f345dca67a4d1dbc886c45154e0ef6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/91f345dca67a4d1dbc886c45154e0ef6 2024-11-17T20:24:18,308 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a472abd7840f4440847a2785820b0c26 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/C/a472abd7840f4440847a2785820b0c26 2024-11-17T20:24:18,311 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/recovered.edits/462.seqid to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811/recovered.edits/462.seqid 2024-11-17T20:24:18,312 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ff04589cbd7acbe489756dc73d4cf811 2024-11-17T20:24:18,312 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T20:24:18,317 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:18,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T20:24:18,326 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T20:24:18,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T20:24:18,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:18,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T20:24:18,355 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731875058355"}]},"ts":"9223372036854775807"} 2024-11-17T20:24:18,359 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T20:24:18,359 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ff04589cbd7acbe489756dc73d4cf811, NAME => 'TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T20:24:18,360 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T20:24:18,360 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731875058360"}]},"ts":"9223372036854775807"} 2024-11-17T20:24:18,363 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T20:24:18,364 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:18,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 86 msec 2024-11-17T20:24:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T20:24:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T20:24:18,390 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:18,400 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;0b0b0ae8de03:46555-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-941759243_22 at /127.0.0.1:59286 [Waiting for operation #226] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1718410319_22 at /127.0.0.1:60264 [Waiting for operation #212] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1718410319_22 at /127.0.0.1:60276 [Waiting for operation #212] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=474 (was 350) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8562 (was 9475) 2024-11-17T20:24:18,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=474, ProcessCount=11, AvailableMemoryMB=8562 2024-11-17T20:24:18,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:24:18,412 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:24:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:18,414 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T20:24:18,414 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:18,415 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-17T20:24:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T20:24:18,416 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T20:24:18,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741939_1115 (size=959) 2024-11-17T20:24:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T20:24:18,660 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T20:24:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T20:24:18,828 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:24:18,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741940_1116 (size=53) 2024-11-17T20:24:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T20:24:19,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:24:19,242 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a6e14f196acc9941f50558a2b30d8d6a, disabling compactions & flushes 2024-11-17T20:24:19,242 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:19,242 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:19,242 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. after waiting 0 ms 2024-11-17T20:24:19,242 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:19,242 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:19,242 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a6e14f196acc9941f50558a2b30d8d6a: Waiting for close lock at 1731875059242Disabling compacts and flushes for region at 1731875059242Disabling writes for close at 1731875059242Writing region close event to WAL at 1731875059242Closed at 1731875059242 2024-11-17T20:24:19,246 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T20:24:19,247 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731875059246"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875059246"}]},"ts":"1731875059246"} 2024-11-17T20:24:19,252 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T20:24:19,254 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T20:24:19,254 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875059254"}]},"ts":"1731875059254"} 2024-11-17T20:24:19,256 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T20:24:19,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, ASSIGN}] 2024-11-17T20:24:19,258 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, ASSIGN 2024-11-17T20:24:19,259 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=false 2024-11-17T20:24:19,410 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:19,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, ASSIGN because future has completed 2024-11-17T20:24:19,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:24:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T20:24:19,551 INFO [AsyncFSWAL-0-hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData-prefix:0b0b0ae8de03,32801,1731875028288 {}] wal.AbstractFSWAL(1368): Slow sync cost: 131 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38093,DS-96187b8f-658b-441b-a2a5-378169f6917f,DISK]] 2024-11-17T20:24:19,707 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:19,708 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:24:19,708 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,709 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:24:19,709 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,709 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,712 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,717 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6e14f196acc9941f50558a2b30d8d6a columnFamilyName A 2024-11-17T20:24:19,717 DEBUG [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:19,718 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(327): Store=a6e14f196acc9941f50558a2b30d8d6a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:19,719 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,721 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6e14f196acc9941f50558a2b30d8d6a columnFamilyName B 2024-11-17T20:24:19,721 DEBUG [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:19,721 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(327): Store=a6e14f196acc9941f50558a2b30d8d6a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:19,721 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,722 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6e14f196acc9941f50558a2b30d8d6a columnFamilyName C 2024-11-17T20:24:19,722 DEBUG [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:19,723 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(327): Store=a6e14f196acc9941f50558a2b30d8d6a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:19,723 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,724 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,724 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,725 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,725 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,726 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:24:19,727 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,729 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:24:19,730 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened a6e14f196acc9941f50558a2b30d8d6a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73011205, jitterRate=0.08795173466205597}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:24:19,730 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:19,731 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for a6e14f196acc9941f50558a2b30d8d6a: Running coprocessor pre-open hook at 1731875059709Writing region info on filesystem at 1731875059709Initializing all the Stores at 1731875059711 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875059711Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875059712 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875059712Cleaning up temporary data from old regions at 1731875059725 (+13 ms)Running coprocessor post-open hooks at 1731875059730 (+5 ms)Region opened successfully at 1731875059731 (+1 ms) 2024-11-17T20:24:19,732 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., pid=36, masterSystemTime=1731875059702 2024-11-17T20:24:19,734 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:19,734 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:19,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:19,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:24:19,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T20:24:19,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 in 322 msec 2024-11-17T20:24:19,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-17T20:24:19,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, ASSIGN in 483 msec 2024-11-17T20:24:19,742 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T20:24:19,742 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875059742"}]},"ts":"1731875059742"} 2024-11-17T20:24:19,744 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T20:24:19,745 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T20:24:19,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3330 sec 2024-11-17T20:24:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T20:24:20,571 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:20,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:20,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:20,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:20,576 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@660ffec3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:20,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@525ba065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:20,582 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:20,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:20,587 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T20:24:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,32801,1731875028288 2024-11-17T20:24:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79383372 2024-11-17T20:24:20,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T20:24:20,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T20:24:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:24:20,593 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:24:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:20,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741941_1117 (size=995) 2024-11-17T20:24:21,012 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T20:24:21,012 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T20:24:21,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:24:21,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, REOPEN/MOVE}] 2024-11-17T20:24:21,023 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, REOPEN/MOVE 2024-11-17T20:24:21,024 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, REOPEN/MOVE because future has completed 2024-11-17T20:24:21,026 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:24:21,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:24:21,179 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,179 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:24:21,179 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing a6e14f196acc9941f50558a2b30d8d6a, disabling compactions & flushes 2024-11-17T20:24:21,179 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,179 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,179 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. after waiting 0 ms 2024-11-17T20:24:21,179 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,183 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T20:24:21,184 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,184 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for a6e14f196acc9941f50558a2b30d8d6a: Waiting for close lock at 1731875061179Running coprocessor pre-close hooks at 1731875061179Disabling compacts and flushes for region at 1731875061179Disabling writes for close at 1731875061179Writing region close event to WAL at 1731875061180 (+1 ms)Running coprocessor post-close hooks at 1731875061184 (+4 ms)Closed at 1731875061184 2024-11-17T20:24:21,184 WARN [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: a6e14f196acc9941f50558a2b30d8d6a to self. 2024-11-17T20:24:21,186 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,187 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=CLOSED 2024-11-17T20:24:21,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:24:21,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-17T20:24:21,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 in 164 msec 2024-11-17T20:24:21,194 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, REOPEN/MOVE; state=CLOSED, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=true 2024-11-17T20:24:21,345 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, REOPEN/MOVE because future has completed 2024-11-17T20:24:21,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:24:21,505 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,505 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:24:21,505 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,506 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:24:21,506 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,506 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,510 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,516 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6e14f196acc9941f50558a2b30d8d6a columnFamilyName A 2024-11-17T20:24:21,518 DEBUG [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:21,519 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(327): Store=a6e14f196acc9941f50558a2b30d8d6a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:21,519 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,520 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6e14f196acc9941f50558a2b30d8d6a columnFamilyName B 2024-11-17T20:24:21,520 DEBUG [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:21,521 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(327): Store=a6e14f196acc9941f50558a2b30d8d6a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:21,521 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,522 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6e14f196acc9941f50558a2b30d8d6a columnFamilyName C 2024-11-17T20:24:21,522 DEBUG [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:21,522 INFO [StoreOpener-a6e14f196acc9941f50558a2b30d8d6a-1 {}] regionserver.HStore(327): Store=a6e14f196acc9941f50558a2b30d8d6a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:21,522 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,523 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,524 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,525 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,525 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,525 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:24:21,527 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,527 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened a6e14f196acc9941f50558a2b30d8d6a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75416860, jitterRate=0.12379878759384155}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:24:21,527 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,528 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for a6e14f196acc9941f50558a2b30d8d6a: Running coprocessor pre-open hook at 1731875061506Writing region info on filesystem at 1731875061506Initializing all the Stores at 1731875061507 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875061507Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875061507Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875061507Cleaning up temporary data from old regions at 1731875061525 (+18 ms)Running coprocessor post-open hooks at 1731875061527 (+2 ms)Region opened successfully at 1731875061528 (+1 ms) 2024-11-17T20:24:21,529 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., pid=41, masterSystemTime=1731875061499 2024-11-17T20:24:21,531 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,531 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,531 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=OPEN, openSeqNum=5, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:24:21,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-17T20:24:21,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 in 187 msec 2024-11-17T20:24:21,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-17T20:24:21,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, REOPEN/MOVE in 513 msec 2024-11-17T20:24:21,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-17T20:24:21,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 523 msec 2024-11-17T20:24:21,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-11-17T20:24:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-17T20:24:21,546 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:21,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4f220f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,550 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574ab902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,552 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228ba1de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145b81d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e17099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdbcba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,562 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a261979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,564 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f092bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,566 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcedf75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,567 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fe8303, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,569 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,569 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e07d977, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,571 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505d96d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,573 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab9db9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144881da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af329d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,578 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fbed63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:21,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:21,580 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261aa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:21,582 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2094988c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:21,583 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,584 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,584 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:21,585 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,585 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T20:24:21,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T20:24:21,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:21,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:21,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:21,590 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,593 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,593 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,594 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:21,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:24:21,613 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:21,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:24:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709c987a3695e4afd868a9a8fea9ae079_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875061599/Put/seqid=0 2024-11-17T20:24:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875121658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875121660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875121662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875121665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875121667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741942_1118 (size=12154) 2024-11-17T20:24:21,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:21,690 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709c987a3695e4afd868a9a8fea9ae079_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709c987a3695e4afd868a9a8fea9ae079_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:21,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7b5f7f3a43bc44a6a6bf44cb3077da43, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:21,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7b5f7f3a43bc44a6a6bf44cb3077da43 is 175, key is test_row_0/A:col10/1731875061599/Put/seqid=0 2024-11-17T20:24:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T20:24:21,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741943_1119 (size=30955) 2024-11-17T20:24:21,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7b5f7f3a43bc44a6a6bf44cb3077da43 2024-11-17T20:24:21,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:21,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:21,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:21,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:21,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ca911ca52061410090825e3a7a3b8cf2 is 50, key is test_row_0/B:col10/1731875061599/Put/seqid=0 2024-11-17T20:24:21,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875121770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875121770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741944_1120 (size=12001) 2024-11-17T20:24:21,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ca911ca52061410090825e3a7a3b8cf2 2024-11-17T20:24:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875121780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875121781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875121781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:21,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/f8af53f690b544038f58e08b64f3e177 is 50, key is test_row_0/C:col10/1731875061599/Put/seqid=0 2024-11-17T20:24:21,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741945_1121 (size=12001) 2024-11-17T20:24:21,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/f8af53f690b544038f58e08b64f3e177 2024-11-17T20:24:21,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7b5f7f3a43bc44a6a6bf44cb3077da43 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7b5f7f3a43bc44a6a6bf44cb3077da43 2024-11-17T20:24:21,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7b5f7f3a43bc44a6a6bf44cb3077da43, entries=150, sequenceid=16, filesize=30.2 K 2024-11-17T20:24:21,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ca911ca52061410090825e3a7a3b8cf2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ca911ca52061410090825e3a7a3b8cf2 2024-11-17T20:24:21,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:21,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:21,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ca911ca52061410090825e3a7a3b8cf2, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T20:24:21,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:21,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/f8af53f690b544038f58e08b64f3e177 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f8af53f690b544038f58e08b64f3e177 2024-11-17T20:24:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T20:24:21,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f8af53f690b544038f58e08b64f3e177, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T20:24:21,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a6e14f196acc9941f50558a2b30d8d6a in 303ms, sequenceid=16, compaction requested=false 2024-11-17T20:24:21,916 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T20:24:21,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:21,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:21,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T20:24:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:22,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875122005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875122008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875122010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875122011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111787a8bb5a5cc446dabc975a424d1c09cc_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875061664/Put/seqid=0 2024-11-17T20:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875122012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741946_1122 (size=17034) 2024-11-17T20:24:22,041 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:22,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111787a8bb5a5cc446dabc975a424d1c09cc_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111787a8bb5a5cc446dabc975a424d1c09cc_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:22,053 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4efb1aed431a4cdaabc94281e6684ad5, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:22,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4efb1aed431a4cdaabc94281e6684ad5 is 175, key is test_row_0/A:col10/1731875061664/Put/seqid=0 2024-11-17T20:24:22,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:22,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:22,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741947_1123 (size=48139) 2024-11-17T20:24:22,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875122119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875122132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875122132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875122132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875122132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T20:24:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875122335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875122351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875122351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875122351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875122351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4efb1aed431a4cdaabc94281e6684ad5 2024-11-17T20:24:22,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/4b5a7d34f10845ddaea550bada0b8224 is 50, key is test_row_0/B:col10/1731875061664/Put/seqid=0 2024-11-17T20:24:22,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741948_1124 (size=12001) 2024-11-17T20:24:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875122650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875122659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875122659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875122670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875122670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T20:24:22,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/4b5a7d34f10845ddaea550bada0b8224 2024-11-17T20:24:22,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/2876939cfb5845a88bbbfd7ee58df316 is 50, key is test_row_0/C:col10/1731875061664/Put/seqid=0 2024-11-17T20:24:22,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741949_1125 (size=12001) 2024-11-17T20:24:22,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/2876939cfb5845a88bbbfd7ee58df316 2024-11-17T20:24:22,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:22,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4efb1aed431a4cdaabc94281e6684ad5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4efb1aed431a4cdaabc94281e6684ad5 2024-11-17T20:24:22,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4efb1aed431a4cdaabc94281e6684ad5, entries=250, sequenceid=42, filesize=47.0 K 2024-11-17T20:24:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/4b5a7d34f10845ddaea550bada0b8224 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4b5a7d34f10845ddaea550bada0b8224 2024-11-17T20:24:23,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4b5a7d34f10845ddaea550bada0b8224, entries=150, sequenceid=42, filesize=11.7 K 2024-11-17T20:24:23,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/2876939cfb5845a88bbbfd7ee58df316 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2876939cfb5845a88bbbfd7ee58df316 2024-11-17T20:24:23,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2876939cfb5845a88bbbfd7ee58df316, entries=150, sequenceid=42, filesize=11.7 K 2024-11-17T20:24:23,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a6e14f196acc9941f50558a2b30d8d6a in 1048ms, sequenceid=42, compaction requested=false 2024-11-17T20:24:23,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:23,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T20:24:23,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:23,138 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T20:24:23,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170d7e65d79a69407294cfc7ce959b99e9_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875062008/Put/seqid=0 2024-11-17T20:24:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:23,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741950_1126 (size=12154) 2024-11-17T20:24:23,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:23,175 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170d7e65d79a69407294cfc7ce959b99e9_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170d7e65d79a69407294cfc7ce959b99e9_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/224ea6bbc748472ca4762bd42c85d84c, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/224ea6bbc748472ca4762bd42c85d84c is 175, key is test_row_0/A:col10/1731875062008/Put/seqid=0 2024-11-17T20:24:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741951_1127 (size=30955) 2024-11-17T20:24:23,209 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/224ea6bbc748472ca4762bd42c85d84c 2024-11-17T20:24:23,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875123209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875123211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875123213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875123215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875123218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ff80358b39df4201b04e3280431ad956 is 50, key is test_row_0/B:col10/1731875062008/Put/seqid=0 2024-11-17T20:24:23,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741952_1128 (size=12001) 2024-11-17T20:24:23,238 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ff80358b39df4201b04e3280431ad956 2024-11-17T20:24:23,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/74b1954533a24a15aa4be25fe04f194b is 50, key is test_row_0/C:col10/1731875062008/Put/seqid=0 2024-11-17T20:24:23,254 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T20:24:23,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741953_1129 (size=12001) 2024-11-17T20:24:23,299 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/74b1954533a24a15aa4be25fe04f194b 2024-11-17T20:24:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/224ea6bbc748472ca4762bd42c85d84c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/224ea6bbc748472ca4762bd42c85d84c 2024-11-17T20:24:23,313 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/224ea6bbc748472ca4762bd42c85d84c, entries=150, sequenceid=52, filesize=30.2 K 2024-11-17T20:24:23,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ff80358b39df4201b04e3280431ad956 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ff80358b39df4201b04e3280431ad956 2024-11-17T20:24:23,322 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ff80358b39df4201b04e3280431ad956, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T20:24:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/74b1954533a24a15aa4be25fe04f194b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/74b1954533a24a15aa4be25fe04f194b 2024-11-17T20:24:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875123319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875123330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875123330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875123331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875123331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,336 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/74b1954533a24a15aa4be25fe04f194b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T20:24:23,338 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a6e14f196acc9941f50558a2b30d8d6a in 199ms, sequenceid=52, compaction requested=true 2024-11-17T20:24:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T20:24:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T20:24:23,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T20:24:23,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7560 sec 2024-11-17T20:24:23,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7680 sec 2024-11-17T20:24:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:23,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:23,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T20:24:23,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dc189406e0a14325a65859fcaed068d6_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875063212/Put/seqid=0 2024-11-17T20:24:23,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875123548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875123549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875123550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875123553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875123555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741954_1130 (size=14594) 2024-11-17T20:24:23,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:23,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dc189406e0a14325a65859fcaed068d6_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dc189406e0a14325a65859fcaed068d6_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:23,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fcfa6eadee224ed5b4f5132550f03976, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:23,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fcfa6eadee224ed5b4f5132550f03976 is 175, key is test_row_0/A:col10/1731875063212/Put/seqid=0 2024-11-17T20:24:23,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741955_1131 (size=39549) 2024-11-17T20:24:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875123659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875123659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875123659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875123670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875123672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T20:24:23,756 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:23,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T20:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T20:24:23,759 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:23,760 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:23,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T20:24:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875123870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875123870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875123870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875123880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875123880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:23,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T20:24:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:23,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fcfa6eadee224ed5b4f5132550f03976 2024-11-17T20:24:24,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/a4c9b3eec14d41fcb2906449f64c9da4 is 50, key is test_row_0/B:col10/1731875063212/Put/seqid=0 2024-11-17T20:24:24,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741956_1132 (size=12001) 2024-11-17T20:24:24,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/a4c9b3eec14d41fcb2906449f64c9da4 2024-11-17T20:24:24,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T20:24:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/66ba0e7cccab4806a95db561c60de397 is 50, key is test_row_0/C:col10/1731875063212/Put/seqid=0 2024-11-17T20:24:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741957_1133 (size=12001) 2024-11-17T20:24:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T20:24:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875124180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875124180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875124180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875124189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875124189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T20:24:24,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:24,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T20:24:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T20:24:24,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/66ba0e7cccab4806a95db561c60de397 2024-11-17T20:24:24,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T20:24:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:24,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fcfa6eadee224ed5b4f5132550f03976 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fcfa6eadee224ed5b4f5132550f03976 2024-11-17T20:24:24,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fcfa6eadee224ed5b4f5132550f03976, entries=200, sequenceid=79, filesize=38.6 K 2024-11-17T20:24:24,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/a4c9b3eec14d41fcb2906449f64c9da4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/a4c9b3eec14d41fcb2906449f64c9da4 2024-11-17T20:24:24,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/a4c9b3eec14d41fcb2906449f64c9da4, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T20:24:24,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/66ba0e7cccab4806a95db561c60de397 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/66ba0e7cccab4806a95db561c60de397 2024-11-17T20:24:24,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/66ba0e7cccab4806a95db561c60de397, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T20:24:24,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a6e14f196acc9941f50558a2b30d8d6a in 1013ms, sequenceid=79, compaction requested=true 2024-11-17T20:24:24,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:24,556 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:24,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:24,557 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:24,558 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:24,558 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:24,558 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,558 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ca911ca52061410090825e3a7a3b8cf2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4b5a7d34f10845ddaea550bada0b8224, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ff80358b39df4201b04e3280431ad956, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/a4c9b3eec14d41fcb2906449f64c9da4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=46.9 K 2024-11-17T20:24:24,559 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:24,559 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:24,559 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,559 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7b5f7f3a43bc44a6a6bf44cb3077da43, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4efb1aed431a4cdaabc94281e6684ad5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/224ea6bbc748472ca4762bd42c85d84c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fcfa6eadee224ed5b4f5132550f03976] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=146.1 K 2024-11-17T20:24:24,559 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,559 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7b5f7f3a43bc44a6a6bf44cb3077da43, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4efb1aed431a4cdaabc94281e6684ad5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/224ea6bbc748472ca4762bd42c85d84c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fcfa6eadee224ed5b4f5132550f03976] 2024-11-17T20:24:24,560 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ca911ca52061410090825e3a7a3b8cf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731875061599 2024-11-17T20:24:24,560 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5f7f3a43bc44a6a6bf44cb3077da43, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731875061599 2024-11-17T20:24:24,561 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b5a7d34f10845ddaea550bada0b8224, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731875061662 2024-11-17T20:24:24,561 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4efb1aed431a4cdaabc94281e6684ad5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731875061657 2024-11-17T20:24:24,561 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ff80358b39df4201b04e3280431ad956, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875062007 2024-11-17T20:24:24,561 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 224ea6bbc748472ca4762bd42c85d84c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875062007 2024-11-17T20:24:24,562 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c9b3eec14d41fcb2906449f64c9da4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731875063204 2024-11-17T20:24:24,562 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcfa6eadee224ed5b4f5132550f03976, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731875063204 2024-11-17T20:24:24,576 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:24,580 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#114 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:24,582 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/9e17c76f5f86458caf6d6995a6f47d51 is 50, key is test_row_0/B:col10/1731875063212/Put/seqid=0 2024-11-17T20:24:24,587 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411171d26ddfb54d2481491a2d3ccd88306c9_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:24,596 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411171d26ddfb54d2481491a2d3ccd88306c9_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:24,597 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171d26ddfb54d2481491a2d3ccd88306c9_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741958_1134 (size=12139) 2024-11-17T20:24:24,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741959_1135 (size=4469) 2024-11-17T20:24:24,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T20:24:24,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,684 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T20:24:24,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ae6770e4d4984b85bc23a191d65d1acb_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875063552/Put/seqid=0 2024-11-17T20:24:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:24,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:24,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741960_1136 (size=12154) 2024-11-17T20:24:24,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:24,743 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ae6770e4d4984b85bc23a191d65d1acb_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae6770e4d4984b85bc23a191d65d1acb_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:24,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/e7e10117506c4f8c8c29a3d840fa41f6, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/e7e10117506c4f8c8c29a3d840fa41f6 is 175, key is test_row_0/A:col10/1731875063552/Put/seqid=0 2024-11-17T20:24:24,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875124737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875124742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875124744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875124748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875124748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741961_1137 (size=30955) 2024-11-17T20:24:24,772 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/e7e10117506c4f8c8c29a3d840fa41f6 2024-11-17T20:24:24,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/55fcc2892ca14e6db14c6f008c60ba2c is 50, key is test_row_0/B:col10/1731875063552/Put/seqid=0 2024-11-17T20:24:24,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741962_1138 (size=12001) 2024-11-17T20:24:24,795 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/55fcc2892ca14e6db14c6f008c60ba2c 2024-11-17T20:24:24,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/f27c3bcdd35f45b883f6c9c679710e46 is 50, key is test_row_0/C:col10/1731875063552/Put/seqid=0 2024-11-17T20:24:24,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741963_1139 (size=12001) 2024-11-17T20:24:24,849 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/f27c3bcdd35f45b883f6c9c679710e46 2024-11-17T20:24:24,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875124850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/e7e10117506c4f8c8c29a3d840fa41f6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/e7e10117506c4f8c8c29a3d840fa41f6 2024-11-17T20:24:24,865 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/e7e10117506c4f8c8c29a3d840fa41f6, entries=150, sequenceid=88, filesize=30.2 K 2024-11-17T20:24:24,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/55fcc2892ca14e6db14c6f008c60ba2c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/55fcc2892ca14e6db14c6f008c60ba2c 2024-11-17T20:24:24,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875124859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875124859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875124870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875124870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:24,876 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/55fcc2892ca14e6db14c6f008c60ba2c, entries=150, sequenceid=88, filesize=11.7 K 2024-11-17T20:24:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/f27c3bcdd35f45b883f6c9c679710e46 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f27c3bcdd35f45b883f6c9c679710e46 2024-11-17T20:24:24,888 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f27c3bcdd35f45b883f6c9c679710e46, entries=150, sequenceid=88, filesize=11.7 K 2024-11-17T20:24:24,891 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for a6e14f196acc9941f50558a2b30d8d6a in 207ms, sequenceid=88, compaction requested=true 2024-11-17T20:24:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T20:24:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T20:24:24,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T20:24:24,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1330 sec 2024-11-17T20:24:24,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1400 sec 2024-11-17T20:24:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T20:24:24,900 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:24,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T20:24:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T20:24:24,903 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:24,904 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:24,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T20:24:25,012 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#115 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:25,013 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/28d4f70966b4468fb052791ca354847b is 175, key is test_row_0/A:col10/1731875063212/Put/seqid=0 2024-11-17T20:24:25,017 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/9e17c76f5f86458caf6d6995a6f47d51 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/9e17c76f5f86458caf6d6995a6f47d51 2024-11-17T20:24:25,024 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into 9e17c76f5f86458caf6d6995a6f47d51(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:25,024 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:25,025 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=12, startTime=1731875064557; duration=0sec 2024-11-17T20:24:25,025 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:25,025 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:25,025 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T20:24:25,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741964_1140 (size=31093) 2024-11-17T20:24:25,034 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/28d4f70966b4468fb052791ca354847b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/28d4f70966b4468fb052791ca354847b 2024-11-17T20:24:25,036 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T20:24:25,037 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/C is initiating minor compaction (all files) 2024-11-17T20:24:25,037 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/C in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:25,037 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f8af53f690b544038f58e08b64f3e177, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2876939cfb5845a88bbbfd7ee58df316, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/74b1954533a24a15aa4be25fe04f194b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/66ba0e7cccab4806a95db561c60de397, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f27c3bcdd35f45b883f6c9c679710e46] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=58.6 K 2024-11-17T20:24:25,040 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f8af53f690b544038f58e08b64f3e177, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731875061599 2024-11-17T20:24:25,041 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2876939cfb5845a88bbbfd7ee58df316, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731875061662 2024-11-17T20:24:25,042 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b1954533a24a15aa4be25fe04f194b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875062007 2024-11-17T20:24:25,043 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 66ba0e7cccab4806a95db561c60de397, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731875063204 2024-11-17T20:24:25,045 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f27c3bcdd35f45b883f6c9c679710e46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731875063545 2024-11-17T20:24:25,046 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 28d4f70966b4468fb052791ca354847b(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:25,046 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:25,046 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=12, startTime=1731875064556; duration=0sec 2024-11-17T20:24:25,046 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:25,046 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:25,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T20:24:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:25,059 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-17T20:24:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:25,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:25,079 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#C#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:25,080 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/7eeef9d742634183901d2f4a5dac2e01 is 50, key is test_row_0/C:col10/1731875063552/Put/seqid=0 2024-11-17T20:24:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875125080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875125081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875125080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875125085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741965_1141 (size=12173) 2024-11-17T20:24:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875125087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175b31da27a2514fe7b477731b2ba23dc8_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875064744/Put/seqid=0 2024-11-17T20:24:25,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741966_1142 (size=12154) 2024-11-17T20:24:25,106 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/7eeef9d742634183901d2f4a5dac2e01 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7eeef9d742634183901d2f4a5dac2e01 2024-11-17T20:24:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:25,116 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175b31da27a2514fe7b477731b2ba23dc8_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b31da27a2514fe7b477731b2ba23dc8_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/36a3f4017e0141768779efb63b4ecf80, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/36a3f4017e0141768779efb63b4ecf80 is 175, key is test_row_0/A:col10/1731875064744/Put/seqid=0 2024-11-17T20:24:25,119 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/C of a6e14f196acc9941f50558a2b30d8d6a into 7eeef9d742634183901d2f4a5dac2e01(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:25,119 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:25,119 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/C, priority=11, startTime=1731875064557; duration=0sec 2024-11-17T20:24:25,119 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:25,119 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:25,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741967_1143 (size=30955) 2024-11-17T20:24:25,128 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/36a3f4017e0141768779efb63b4ecf80 2024-11-17T20:24:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/17fdbda036df4175b5107d45c969cec9 is 50, key is test_row_0/B:col10/1731875064744/Put/seqid=0 2024-11-17T20:24:25,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741968_1144 (size=12001) 2024-11-17T20:24:25,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875125190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875125199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T20:24:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875125390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875125390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875125399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875125399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875125410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T20:24:25,559 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/17fdbda036df4175b5107d45c969cec9 2024-11-17T20:24:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/11f85a588a4b4957920a8df932f9db4a is 50, key is test_row_0/C:col10/1731875064744/Put/seqid=0 2024-11-17T20:24:25,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741969_1145 (size=12001) 2024-11-17T20:24:25,587 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/11f85a588a4b4957920a8df932f9db4a 2024-11-17T20:24:25,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/36a3f4017e0141768779efb63b4ecf80 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/36a3f4017e0141768779efb63b4ecf80 2024-11-17T20:24:25,599 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/36a3f4017e0141768779efb63b4ecf80, entries=150, sequenceid=118, filesize=30.2 K 2024-11-17T20:24:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/17fdbda036df4175b5107d45c969cec9 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/17fdbda036df4175b5107d45c969cec9 2024-11-17T20:24:25,607 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/17fdbda036df4175b5107d45c969cec9, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T20:24:25,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/11f85a588a4b4957920a8df932f9db4a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/11f85a588a4b4957920a8df932f9db4a 2024-11-17T20:24:25,616 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/11f85a588a4b4957920a8df932f9db4a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T20:24:25,617 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for a6e14f196acc9941f50558a2b30d8d6a in 558ms, sequenceid=118, compaction requested=true 2024-11-17T20:24:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T20:24:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T20:24:25,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T20:24:25,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 715 msec 2024-11-17T20:24:25,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 723 msec 2024-11-17T20:24:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:25,724 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:25,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:24:25,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7acccb528364ac8ba9c0bcf0e79e95a_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875065723/Put/seqid=0 2024-11-17T20:24:25,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741970_1146 (size=17134) 2024-11-17T20:24:25,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:25,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7acccb528364ac8ba9c0bcf0e79e95a_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7acccb528364ac8ba9c0bcf0e79e95a_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:25,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f44b819f3c1c4a54ac766a749d200567, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:25,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f44b819f3c1c4a54ac766a749d200567 is 175, key is test_row_0/A:col10/1731875065723/Put/seqid=0 2024-11-17T20:24:25,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741971_1147 (size=48239) 2024-11-17T20:24:25,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875125813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875125820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875125900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875125900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875125909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875125930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875125940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:25,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T20:24:26,040 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:26,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T20:24:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T20:24:26,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:26,046 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:26,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T20:24:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875126149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875126149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f44b819f3c1c4a54ac766a749d200567 2024-11-17T20:24:26,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/26e9ff9b1a134357b1a0a7a3f31a1f7b is 50, key is test_row_0/B:col10/1731875065723/Put/seqid=0 2024-11-17T20:24:26,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741972_1148 (size=12101) 2024-11-17T20:24:26,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T20:24:26,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875126460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875126460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:26,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:26,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:26,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T20:24:26,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/26e9ff9b1a134357b1a0a7a3f31a1f7b 2024-11-17T20:24:26,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5e8ebaf65ac34a3dae13b5462e4fe23d is 50, key is test_row_0/C:col10/1731875065723/Put/seqid=0 2024-11-17T20:24:26,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741973_1149 (size=12101) 2024-11-17T20:24:26,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875126920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875126920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,932 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,930 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875126937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,941 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:26,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875126970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:26,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875126979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:27,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:27,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5e8ebaf65ac34a3dae13b5462e4fe23d 2024-11-17T20:24:27,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f44b819f3c1c4a54ac766a749d200567 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f44b819f3c1c4a54ac766a749d200567 2024-11-17T20:24:27,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f44b819f3c1c4a54ac766a749d200567, entries=250, sequenceid=130, filesize=47.1 K 2024-11-17T20:24:27,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/26e9ff9b1a134357b1a0a7a3f31a1f7b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/26e9ff9b1a134357b1a0a7a3f31a1f7b 2024-11-17T20:24:27,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/26e9ff9b1a134357b1a0a7a3f31a1f7b, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T20:24:27,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5e8ebaf65ac34a3dae13b5462e4fe23d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5e8ebaf65ac34a3dae13b5462e4fe23d 2024-11-17T20:24:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T20:24:27,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5e8ebaf65ac34a3dae13b5462e4fe23d, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T20:24:27,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a6e14f196acc9941f50558a2b30d8d6a in 1466ms, sequenceid=130, compaction requested=true 2024-11-17T20:24:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:27,190 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:27,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:27,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:27,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:27,191 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:27,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:27,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:27,194 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:27,194 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:27,195 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:27,195 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,195 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:27,195 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,198 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/28d4f70966b4468fb052791ca354847b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/e7e10117506c4f8c8c29a3d840fa41f6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/36a3f4017e0141768779efb63b4ecf80, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f44b819f3c1c4a54ac766a749d200567] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=137.9 K 2024-11-17T20:24:27,198 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,198 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/28d4f70966b4468fb052791ca354847b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/e7e10117506c4f8c8c29a3d840fa41f6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/36a3f4017e0141768779efb63b4ecf80, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f44b819f3c1c4a54ac766a749d200567] 2024-11-17T20:24:27,195 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/9e17c76f5f86458caf6d6995a6f47d51, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/55fcc2892ca14e6db14c6f008c60ba2c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/17fdbda036df4175b5107d45c969cec9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/26e9ff9b1a134357b1a0a7a3f31a1f7b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=47.1 K 2024-11-17T20:24:27,199 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d4f70966b4468fb052791ca354847b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731875063204 2024-11-17T20:24:27,199 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e17c76f5f86458caf6d6995a6f47d51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731875063204 2024-11-17T20:24:27,200 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7e10117506c4f8c8c29a3d840fa41f6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731875063545 2024-11-17T20:24:27,200 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 55fcc2892ca14e6db14c6f008c60ba2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731875063545 2024-11-17T20:24:27,200 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a3f4017e0141768779efb63b4ecf80, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731875064717 2024-11-17T20:24:27,200 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 17fdbda036df4175b5107d45c969cec9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731875064717 2024-11-17T20:24:27,200 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44b819f3c1c4a54ac766a749d200567, keycount=250, bloomtype=ROW, size=47.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875065068 2024-11-17T20:24:27,200 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e9ff9b1a134357b1a0a7a3f31a1f7b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875065083 2024-11-17T20:24:27,221 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:27,224 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#127 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:27,224 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/d3e45b22fb1a455e90705d3210676d00 is 50, key is test_row_0/B:col10/1731875065723/Put/seqid=0 2024-11-17T20:24:27,233 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111739c025ea69b141ae8d59a67c2e87b54b_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:27,239 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111739c025ea69b141ae8d59a67c2e87b54b_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:27,239 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111739c025ea69b141ae8d59a67c2e87b54b_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:27,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741974_1150 (size=12375) 2024-11-17T20:24:27,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T20:24:27,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,289 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:24:27,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741975_1151 (size=4469) 2024-11-17T20:24:27,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ad22da1cad64737b970d467bab1abb2_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875065818/Put/seqid=0 2024-11-17T20:24:27,324 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#126 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:27,325 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/3d6d4debe1b245c4ac5d97082d11089d is 175, key is test_row_0/A:col10/1731875065723/Put/seqid=0 2024-11-17T20:24:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741977_1153 (size=31329) 2024-11-17T20:24:27,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741976_1152 (size=12304) 2024-11-17T20:24:27,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,373 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ad22da1cad64737b970d467bab1abb2_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ad22da1cad64737b970d467bab1abb2_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:27,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/a2ecf3f6ad0149e5a4d2f9893001db63, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:27,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/a2ecf3f6ad0149e5a4d2f9893001db63 is 175, key is test_row_0/A:col10/1731875065818/Put/seqid=0 2024-11-17T20:24:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741978_1154 (size=31105) 2024-11-17T20:24:27,406 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/a2ecf3f6ad0149e5a4d2f9893001db63 2024-11-17T20:24:27,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/4df57ab963304ce388ac7fce72e630e3 is 50, key is test_row_0/B:col10/1731875065818/Put/seqid=0 2024-11-17T20:24:27,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741979_1155 (size=12151) 2024-11-17T20:24:27,447 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/4df57ab963304ce388ac7fce72e630e3 2024-11-17T20:24:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/2efb58b513344cd38e4ad28f96a9bec4 is 50, key is test_row_0/C:col10/1731875065818/Put/seqid=0 2024-11-17T20:24:27,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741980_1156 (size=12151) 2024-11-17T20:24:27,488 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/2efb58b513344cd38e4ad28f96a9bec4 2024-11-17T20:24:27,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/a2ecf3f6ad0149e5a4d2f9893001db63 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/a2ecf3f6ad0149e5a4d2f9893001db63 2024-11-17T20:24:27,505 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/a2ecf3f6ad0149e5a4d2f9893001db63, entries=150, sequenceid=155, filesize=30.4 K 2024-11-17T20:24:27,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/4df57ab963304ce388ac7fce72e630e3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4df57ab963304ce388ac7fce72e630e3 2024-11-17T20:24:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,514 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4df57ab963304ce388ac7fce72e630e3, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T20:24:27,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/2efb58b513344cd38e4ad28f96a9bec4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2efb58b513344cd38e4ad28f96a9bec4 2024-11-17T20:24:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,540 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2efb58b513344cd38e4ad28f96a9bec4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T20:24:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,542 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for a6e14f196acc9941f50558a2b30d8d6a in 252ms, sequenceid=155, compaction requested=true 2024-11-17T20:24:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T20:24:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T20:24:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T20:24:27,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-17T20:24:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5050 sec 2024-11-17T20:24:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,677 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/d3e45b22fb1a455e90705d3210676d00 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d3e45b22fb1a455e90705d3210676d00 2024-11-17T20:24:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,685 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into d3e45b22fb1a455e90705d3210676d00(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:27,685 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:27,685 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=12, startTime=1731875067191; duration=0sec 2024-11-17T20:24:27,685 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:27,685 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:27,685 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,688 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48426 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:27,688 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/C is initiating minor compaction (all files) 2024-11-17T20:24:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,689 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/C in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:27,689 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7eeef9d742634183901d2f4a5dac2e01, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/11f85a588a4b4957920a8df932f9db4a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5e8ebaf65ac34a3dae13b5462e4fe23d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2efb58b513344cd38e4ad28f96a9bec4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=47.3 K 2024-11-17T20:24:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,689 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eeef9d742634183901d2f4a5dac2e01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731875063545 2024-11-17T20:24:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,690 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f85a588a4b4957920a8df932f9db4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731875064717 2024-11-17T20:24:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,690 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8ebaf65ac34a3dae13b5462e4fe23d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875065083 2024-11-17T20:24:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,691 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efb58b513344cd38e4ad28f96a9bec4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731875065778 2024-11-17T20:24:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,710 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,711 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/4fa5177985f64b11ade9e619aaf99bdb is 50, key is test_row_0/C:col10/1731875065818/Put/seqid=0 2024-11-17T20:24:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741981_1157 (size=12459) 2024-11-17T20:24:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,774 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/3d6d4debe1b245c4ac5d97082d11089d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/3d6d4debe1b245c4ac5d97082d11089d 2024-11-17T20:24:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,781 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 3d6d4debe1b245c4ac5d97082d11089d(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:27,781 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:27,781 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=12, startTime=1731875067190; duration=0sec 2024-11-17T20:24:27,781 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:27,781 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:28,086 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:28,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:24:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174a7f8b0d74e04191a59b6e8512beeaa6_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875068075/Put/seqid=0 2024-11-17T20:24:28,147 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/4fa5177985f64b11ade9e619aaf99bdb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/4fa5177985f64b11ade9e619aaf99bdb 2024-11-17T20:24:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741983_1159 (size=24758) 2024-11-17T20:24:28,156 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/C of a6e14f196acc9941f50558a2b30d8d6a into 4fa5177985f64b11ade9e619aaf99bdb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:28,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:28,156 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/C, priority=12, startTime=1731875067191; duration=0sec 2024-11-17T20:24:28,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:28,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:28,161 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875128159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875128165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,169 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174a7f8b0d74e04191a59b6e8512beeaa6_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174a7f8b0d74e04191a59b6e8512beeaa6_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:28,170 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4077886b070d4395a354ec6191cbc9c0, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:28,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4077886b070d4395a354ec6191cbc9c0 is 175, key is test_row_0/A:col10/1731875068075/Put/seqid=0 2024-11-17T20:24:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741982_1158 (size=74395) 2024-11-17T20:24:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T20:24:28,189 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:28,190 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T20:24:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T20:24:28,192 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:28,193 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:28,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:28,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875128271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875128279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T20:24:28,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T20:24:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875128479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875128489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T20:24:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T20:24:28,574 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4077886b070d4395a354ec6191cbc9c0 2024-11-17T20:24:28,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6e662f43a6524eb4931b08e2e3fb1907 is 50, key is test_row_0/B:col10/1731875068075/Put/seqid=0 2024-11-17T20:24:28,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741984_1160 (size=12151) 2024-11-17T20:24:28,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T20:24:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,660 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T20:24:28,660 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T20:24:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875128789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875128799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T20:24:28,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:28,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T20:24:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875128950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875128950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,954 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,954 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875128960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:28,962 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:28,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T20:24:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:28,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:28,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6e662f43a6524eb4931b08e2e3fb1907 2024-11-17T20:24:28,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/42b540e8517046799f7c72bdc7f01506 is 50, key is test_row_0/C:col10/1731875068075/Put/seqid=0 2024-11-17T20:24:29,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741985_1161 (size=12151) 2024-11-17T20:24:29,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/42b540e8517046799f7c72bdc7f01506 2024-11-17T20:24:29,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/4077886b070d4395a354ec6191cbc9c0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4077886b070d4395a354ec6191cbc9c0 2024-11-17T20:24:29,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4077886b070d4395a354ec6191cbc9c0, entries=400, sequenceid=169, filesize=72.7 K 2024-11-17T20:24:29,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6e662f43a6524eb4931b08e2e3fb1907 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e662f43a6524eb4931b08e2e3fb1907 2024-11-17T20:24:29,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e662f43a6524eb4931b08e2e3fb1907, entries=150, sequenceid=169, filesize=11.9 K 2024-11-17T20:24:29,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/42b540e8517046799f7c72bdc7f01506 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/42b540e8517046799f7c72bdc7f01506 2024-11-17T20:24:29,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/42b540e8517046799f7c72bdc7f01506, entries=150, sequenceid=169, filesize=11.9 K 2024-11-17T20:24:29,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a6e14f196acc9941f50558a2b30d8d6a in 966ms, sequenceid=169, compaction requested=true 2024-11-17T20:24:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:29,052 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:29,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:29,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:29,054 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:29,054 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:29,054 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:29,055 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:29,055 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/3d6d4debe1b245c4ac5d97082d11089d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/a2ecf3f6ad0149e5a4d2f9893001db63, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4077886b070d4395a354ec6191cbc9c0] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=133.6 K 2024-11-17T20:24:29,055 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:29,055 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/3d6d4debe1b245c4ac5d97082d11089d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/a2ecf3f6ad0149e5a4d2f9893001db63, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4077886b070d4395a354ec6191cbc9c0] 2024-11-17T20:24:29,055 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:29,055 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:29,055 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:29,055 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d3e45b22fb1a455e90705d3210676d00, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4df57ab963304ce388ac7fce72e630e3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e662f43a6524eb4931b08e2e3fb1907] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=35.8 K 2024-11-17T20:24:29,056 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d6d4debe1b245c4ac5d97082d11089d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875065083 2024-11-17T20:24:29,058 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e45b22fb1a455e90705d3210676d00, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875065083 2024-11-17T20:24:29,058 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ecf3f6ad0149e5a4d2f9893001db63, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731875065778 2024-11-17T20:24:29,059 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4df57ab963304ce388ac7fce72e630e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731875065778 2024-11-17T20:24:29,059 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e662f43a6524eb4931b08e2e3fb1907, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731875068060 2024-11-17T20:24:29,061 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4077886b070d4395a354ec6191cbc9c0, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731875068024 2024-11-17T20:24:29,069 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:29,070 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6e099493873a46c492d88b4cc7d5bcb9 is 50, key is test_row_0/B:col10/1731875068075/Put/seqid=0 2024-11-17T20:24:29,089 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:29,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741986_1162 (size=12527) 2024-11-17T20:24:29,096 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411172416534e85ff47008660f79577cb9ab1_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:29,100 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411172416534e85ff47008660f79577cb9ab1_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:29,101 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172416534e85ff47008660f79577cb9ab1_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:29,108 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6e099493873a46c492d88b4cc7d5bcb9 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e099493873a46c492d88b4cc7d5bcb9 2024-11-17T20:24:29,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T20:24:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:29,121 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:24:29,121 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into 6e099493873a46c492d88b4cc7d5bcb9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:29,121 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:29,121 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=13, startTime=1731875069054; duration=0sec 2024-11-17T20:24:29,121 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:29,123 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:29,123 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T20:24:29,124 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T20:24:29,125 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T20:24:29,125 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. because compaction request was cancelled 2024-11-17T20:24:29,125 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:29,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741987_1163 (size=4469) 2024-11-17T20:24:29,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e223cfe4df9945a399a6dc8ead00224f_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875068156/Put/seqid=0 2024-11-17T20:24:29,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741988_1164 (size=12304) 2024-11-17T20:24:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:29,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T20:24:29,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875129331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875129331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875129439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875129450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,585 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#136 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:29,586 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/71e929ea336c4878aa7358d817f12933 is 175, key is test_row_0/A:col10/1731875068075/Put/seqid=0 2024-11-17T20:24:29,590 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e223cfe4df9945a399a6dc8ead00224f_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e223cfe4df9945a399a6dc8ead00224f_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:29,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741989_1165 (size=31481) 2024-11-17T20:24:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/ea69a1edb716466790745782d7ff9d7a, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/ea69a1edb716466790745782d7ff9d7a is 175, key is test_row_0/A:col10/1731875068156/Put/seqid=0 2024-11-17T20:24:29,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741990_1166 (size=31105) 2024-11-17T20:24:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875129649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875129659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875129959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875129969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:29,996 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/ea69a1edb716466790745782d7ff9d7a 2024-11-17T20:24:29,997 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/71e929ea336c4878aa7358d817f12933 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/71e929ea336c4878aa7358d817f12933 2024-11-17T20:24:30,003 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 71e929ea336c4878aa7358d817f12933(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:30,003 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:30,003 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=13, startTime=1731875069052; duration=0sec 2024-11-17T20:24:30,003 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:30,003 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:30,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/c27d2a94c4464accbf18d5ea2b62ebfd is 50, key is test_row_0/B:col10/1731875068156/Put/seqid=0 2024-11-17T20:24:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741991_1167 (size=12151) 2024-11-17T20:24:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T20:24:30,411 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/c27d2a94c4464accbf18d5ea2b62ebfd 2024-11-17T20:24:30,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/34d8b12dbf174ff3a069717673bf0684 is 50, key is test_row_0/C:col10/1731875068156/Put/seqid=0 2024-11-17T20:24:30,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741992_1168 (size=12151) 2024-11-17T20:24:30,426 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/34d8b12dbf174ff3a069717673bf0684 2024-11-17T20:24:30,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/ea69a1edb716466790745782d7ff9d7a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/ea69a1edb716466790745782d7ff9d7a 2024-11-17T20:24:30,435 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/ea69a1edb716466790745782d7ff9d7a, entries=150, sequenceid=195, filesize=30.4 K 2024-11-17T20:24:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/c27d2a94c4464accbf18d5ea2b62ebfd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/c27d2a94c4464accbf18d5ea2b62ebfd 2024-11-17T20:24:30,440 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/c27d2a94c4464accbf18d5ea2b62ebfd, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T20:24:30,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/34d8b12dbf174ff3a069717673bf0684 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/34d8b12dbf174ff3a069717673bf0684 2024-11-17T20:24:30,447 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/34d8b12dbf174ff3a069717673bf0684, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T20:24:30,448 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a6e14f196acc9941f50558a2b30d8d6a in 1328ms, sequenceid=195, compaction requested=true 2024-11-17T20:24:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T20:24:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T20:24:30,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T20:24:30,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2550 sec 2024-11-17T20:24:30,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.2610 sec 2024-11-17T20:24:30,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:30,470 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:30,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:24:30,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176b86648e26d344a9a6446c7febe74256_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875069329/Put/seqid=0 2024-11-17T20:24:30,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741993_1169 (size=14794) 2024-11-17T20:24:30,496 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:30,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176b86648e26d344a9a6446c7febe74256_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176b86648e26d344a9a6446c7febe74256_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:30,505 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d16f980f605c47d682f5b8f842952f3b, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:30,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d16f980f605c47d682f5b8f842952f3b is 175, key is test_row_0/A:col10/1731875069329/Put/seqid=0 2024-11-17T20:24:30,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741994_1170 (size=39749) 2024-11-17T20:24:30,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875130521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:30,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875130524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875130630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:30,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875130639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:30,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875130840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:30,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875130850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:30,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:30,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d16f980f605c47d682f5b8f842952f3b 2024-11-17T20:24:30,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0ac3f310e98e4583a37734eb5659492d is 50, key is test_row_0/B:col10/1731875069329/Put/seqid=0 2024-11-17T20:24:30,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741995_1171 (size=12151) 2024-11-17T20:24:30,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0ac3f310e98e4583a37734eb5659492d 2024-11-17T20:24:30,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/625aea588cfb492ea07b1bf4ee085129 is 50, key is test_row_0/C:col10/1731875069329/Put/seqid=0 2024-11-17T20:24:30,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741996_1172 (size=12151) 2024-11-17T20:24:30,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/625aea588cfb492ea07b1bf4ee085129 2024-11-17T20:24:30,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d16f980f605c47d682f5b8f842952f3b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d16f980f605c47d682f5b8f842952f3b 2024-11-17T20:24:30,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d16f980f605c47d682f5b8f842952f3b, entries=200, sequenceid=210, filesize=38.8 K 2024-11-17T20:24:30,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0ac3f310e98e4583a37734eb5659492d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0ac3f310e98e4583a37734eb5659492d 2024-11-17T20:24:31,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0ac3f310e98e4583a37734eb5659492d, entries=150, sequenceid=210, filesize=11.9 K 2024-11-17T20:24:31,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/625aea588cfb492ea07b1bf4ee085129 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/625aea588cfb492ea07b1bf4ee085129 2024-11-17T20:24:31,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/625aea588cfb492ea07b1bf4ee085129, entries=150, sequenceid=210, filesize=11.9 K 2024-11-17T20:24:31,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a6e14f196acc9941f50558a2b30d8d6a in 536ms, sequenceid=210, compaction requested=true 2024-11-17T20:24:31,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:31,007 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:31,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:31,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:31,008 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:31,010 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:31,010 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:31,010 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:31,010 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/71e929ea336c4878aa7358d817f12933, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/ea69a1edb716466790745782d7ff9d7a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d16f980f605c47d682f5b8f842952f3b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=99.9 K 2024-11-17T20:24:31,010 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:31,010 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/71e929ea336c4878aa7358d817f12933, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/ea69a1edb716466790745782d7ff9d7a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d16f980f605c47d682f5b8f842952f3b] 2024-11-17T20:24:31,011 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:31,011 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e929ea336c4878aa7358d817f12933, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731875068060 2024-11-17T20:24:31,011 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:31,011 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:31,011 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e099493873a46c492d88b4cc7d5bcb9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/c27d2a94c4464accbf18d5ea2b62ebfd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0ac3f310e98e4583a37734eb5659492d] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=36.0 K 2024-11-17T20:24:31,012 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e099493873a46c492d88b4cc7d5bcb9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731875068060 2024-11-17T20:24:31,012 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea69a1edb716466790745782d7ff9d7a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731875068134 2024-11-17T20:24:31,013 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c27d2a94c4464accbf18d5ea2b62ebfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731875068134 2024-11-17T20:24:31,014 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d16f980f605c47d682f5b8f842952f3b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875069321 2024-11-17T20:24:31,015 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ac3f310e98e4583a37734eb5659492d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875069328 2024-11-17T20:24:31,023 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:31,038 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:31,039 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117690711b6596b4bd7837e37db3b481265_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:31,039 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/40090ff30a20439c9adb20bda7f70a39 is 50, key is test_row_0/B:col10/1731875069329/Put/seqid=0 2024-11-17T20:24:31,041 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117690711b6596b4bd7837e37db3b481265_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:31,041 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117690711b6596b4bd7837e37db3b481265_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:31,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741997_1173 (size=12629) 2024-11-17T20:24:31,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741998_1174 (size=4469) 2024-11-17T20:24:31,068 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/40090ff30a20439c9adb20bda7f70a39 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/40090ff30a20439c9adb20bda7f70a39 2024-11-17T20:24:31,069 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#143 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:31,070 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/15904128924a47719f8090ecca207369 is 175, key is test_row_0/A:col10/1731875069329/Put/seqid=0 2024-11-17T20:24:31,077 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into 40090ff30a20439c9adb20bda7f70a39(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:31,077 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:31,077 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=13, startTime=1731875071007; duration=0sec 2024-11-17T20:24:31,077 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:31,077 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:31,077 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:31,079 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:31,079 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/C is initiating minor compaction (all files) 2024-11-17T20:24:31,079 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/C in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:31,080 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/4fa5177985f64b11ade9e619aaf99bdb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/42b540e8517046799f7c72bdc7f01506, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/34d8b12dbf174ff3a069717673bf0684, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/625aea588cfb492ea07b1bf4ee085129] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=47.8 K 2024-11-17T20:24:31,080 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa5177985f64b11ade9e619aaf99bdb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731875065778 2024-11-17T20:24:31,080 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b540e8517046799f7c72bdc7f01506, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731875068060 2024-11-17T20:24:31,081 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d8b12dbf174ff3a069717673bf0684, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731875068134 2024-11-17T20:24:31,081 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 625aea588cfb492ea07b1bf4ee085129, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875069328 2024-11-17T20:24:31,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741999_1175 (size=31583) 2024-11-17T20:24:31,099 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#C#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:31,100 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5c252dca23374902be3f36196907a039 is 50, key is test_row_0/C:col10/1731875069329/Put/seqid=0 2024-11-17T20:24:31,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742000_1176 (size=12595) 2024-11-17T20:24:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:31,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:31,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T20:24:31,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117754c3c62251145e6ad1ad7010ae06e2b_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875070517/Put/seqid=0 2024-11-17T20:24:31,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875131183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:31,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875131185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:31,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742001_1177 (size=14794) 2024-11-17T20:24:31,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875131290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:31,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875131299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:31,501 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/15904128924a47719f8090ecca207369 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/15904128924a47719f8090ecca207369 2024-11-17T20:24:31,506 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 15904128924a47719f8090ecca207369(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:31,506 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:31,506 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=13, startTime=1731875071007; duration=0sec 2024-11-17T20:24:31,506 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:31,506 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:31,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875131500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:31,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875131509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:31,525 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5c252dca23374902be3f36196907a039 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5c252dca23374902be3f36196907a039 2024-11-17T20:24:31,533 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/C of a6e14f196acc9941f50558a2b30d8d6a into 5c252dca23374902be3f36196907a039(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:31,533 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:31,534 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/C, priority=12, startTime=1731875071008; duration=0sec 2024-11-17T20:24:31,534 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:31,534 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:31,598 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:31,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117754c3c62251145e6ad1ad7010ae06e2b_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117754c3c62251145e6ad1ad7010ae06e2b_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:31,606 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d9709e7a08e1447db060a0d6a1a8c694, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:31,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d9709e7a08e1447db060a0d6a1a8c694 is 175, key is test_row_0/A:col10/1731875070517/Put/seqid=0 2024-11-17T20:24:31,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742002_1178 (size=39749) 2024-11-17T20:24:31,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875131819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875131819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:32,024 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d9709e7a08e1447db060a0d6a1a8c694 2024-11-17T20:24:32,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/16c2fa5f8a9340e2831eb8c6fcdcab51 is 50, key is test_row_0/B:col10/1731875070517/Put/seqid=0 2024-11-17T20:24:32,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742003_1179 (size=12151) 2024-11-17T20:24:32,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/16c2fa5f8a9340e2831eb8c6fcdcab51 2024-11-17T20:24:32,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/568fe00fed0c4585ae3f6008b8085283 is 50, key is test_row_0/C:col10/1731875070517/Put/seqid=0 2024-11-17T20:24:32,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742004_1180 (size=12151) 2024-11-17T20:24:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T20:24:32,359 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:32,360 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T20:24:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875132357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875132357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T20:24:32,364 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:32,365 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:32,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T20:24:32,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/568fe00fed0c4585ae3f6008b8085283 2024-11-17T20:24:32,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/d9709e7a08e1447db060a0d6a1a8c694 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d9709e7a08e1447db060a0d6a1a8c694 2024-11-17T20:24:32,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d9709e7a08e1447db060a0d6a1a8c694, entries=200, sequenceid=234, filesize=38.8 K 2024-11-17T20:24:32,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/16c2fa5f8a9340e2831eb8c6fcdcab51 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/16c2fa5f8a9340e2831eb8c6fcdcab51 2024-11-17T20:24:32,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/16c2fa5f8a9340e2831eb8c6fcdcab51, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T20:24:32,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/568fe00fed0c4585ae3f6008b8085283 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/568fe00fed0c4585ae3f6008b8085283 2024-11-17T20:24:32,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T20:24:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:32,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:32,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/568fe00fed0c4585ae3f6008b8085283, entries=150, sequenceid=234, filesize=11.9 K 2024-11-17T20:24:32,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a6e14f196acc9941f50558a2b30d8d6a in 1378ms, sequenceid=234, compaction requested=false 2024-11-17T20:24:32,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:32,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T20:24:32,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:32,674 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T20:24:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T20:24:32,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ddc0dcc809b54ff6a99832127daa9a33_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875071183/Put/seqid=0 2024-11-17T20:24:32,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742005_1181 (size=12304) 2024-11-17T20:24:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T20:24:33,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875133009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875133015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875133016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:33,099 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ddc0dcc809b54ff6a99832127daa9a33_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ddc0dcc809b54ff6a99832127daa9a33_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:33,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f0f032a8400b4e40bd2ee0a3b1e0b889, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f0f032a8400b4e40bd2ee0a3b1e0b889 is 175, key is test_row_0/A:col10/1731875071183/Put/seqid=0 2024-11-17T20:24:33,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742006_1182 (size=31105) 2024-11-17T20:24:33,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875133119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875133129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875133129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875133329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875133339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875133340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875133380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,384 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875133386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,388 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,424 INFO [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T20:24:33,424 INFO [master/0b0b0ae8de03:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T20:24:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T20:24:33,518 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f0f032a8400b4e40bd2ee0a3b1e0b889 2024-11-17T20:24:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/abc5dcfba0714dd6a5ade7503a8ae2c1 is 50, key is test_row_0/B:col10/1731875071183/Put/seqid=0 2024-11-17T20:24:33,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742007_1183 (size=12151) 2024-11-17T20:24:33,539 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/abc5dcfba0714dd6a5ade7503a8ae2c1 2024-11-17T20:24:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/1d7b98462a0941a2936feddb598a6f5a is 50, key is test_row_0/C:col10/1731875071183/Put/seqid=0 2024-11-17T20:24:33,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742008_1184 (size=12151) 2024-11-17T20:24:33,567 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/1d7b98462a0941a2936feddb598a6f5a 2024-11-17T20:24:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/f0f032a8400b4e40bd2ee0a3b1e0b889 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f0f032a8400b4e40bd2ee0a3b1e0b889 2024-11-17T20:24:33,579 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f0f032a8400b4e40bd2ee0a3b1e0b889, entries=150, sequenceid=249, filesize=30.4 K 2024-11-17T20:24:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/abc5dcfba0714dd6a5ade7503a8ae2c1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/abc5dcfba0714dd6a5ade7503a8ae2c1 2024-11-17T20:24:33,588 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/abc5dcfba0714dd6a5ade7503a8ae2c1, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T20:24:33,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/1d7b98462a0941a2936feddb598a6f5a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/1d7b98462a0941a2936feddb598a6f5a 2024-11-17T20:24:33,596 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/1d7b98462a0941a2936feddb598a6f5a, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T20:24:33,597 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a6e14f196acc9941f50558a2b30d8d6a in 923ms, sequenceid=249, compaction requested=true 2024-11-17T20:24:33,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:33,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:33,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T20:24:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T20:24:33,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T20:24:33,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2330 sec 2024-11-17T20:24:33,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.2410 sec 2024-11-17T20:24:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:33,641 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:33,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:24:33,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c5412c0046f7469295abf66dfdbb057e_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875073014/Put/seqid=0 2024-11-17T20:24:33,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742009_1185 (size=14994) 2024-11-17T20:24:33,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:33,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c5412c0046f7469295abf66dfdbb057e_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c5412c0046f7469295abf66dfdbb057e_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:33,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/63fffd84bb03487b92186cb8d130aac8, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:33,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/63fffd84bb03487b92186cb8d130aac8 is 175, key is test_row_0/A:col10/1731875073014/Put/seqid=0 2024-11-17T20:24:33,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875133660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875133661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875133662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742010_1186 (size=39949) 2024-11-17T20:24:33,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875133769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875133779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875133779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875133979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875133989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875133989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:34,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/63fffd84bb03487b92186cb8d130aac8 2024-11-17T20:24:34,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0a6efddf0e79425a94665f7c7e524a6e is 50, key is test_row_0/B:col10/1731875073014/Put/seqid=0 2024-11-17T20:24:34,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742011_1187 (size=12301) 2024-11-17T20:24:34,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0a6efddf0e79425a94665f7c7e524a6e 2024-11-17T20:24:34,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/d3e16fab48b9468b85a9c727cd9f1de0 is 50, key is test_row_0/C:col10/1731875073014/Put/seqid=0 2024-11-17T20:24:34,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742012_1188 (size=12301) 2024-11-17T20:24:34,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875134290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875134299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875134299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T20:24:34,510 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:34,511 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-17T20:24:34,514 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:34,515 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:34,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T20:24:34,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/d3e16fab48b9468b85a9c727cd9f1de0 2024-11-17T20:24:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T20:24:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/63fffd84bb03487b92186cb8d130aac8 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/63fffd84bb03487b92186cb8d130aac8 2024-11-17T20:24:34,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/63fffd84bb03487b92186cb8d130aac8, entries=200, sequenceid=273, filesize=39.0 K 2024-11-17T20:24:34,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0a6efddf0e79425a94665f7c7e524a6e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0a6efddf0e79425a94665f7c7e524a6e 2024-11-17T20:24:34,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0a6efddf0e79425a94665f7c7e524a6e, entries=150, sequenceid=273, filesize=12.0 K 2024-11-17T20:24:34,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/d3e16fab48b9468b85a9c727cd9f1de0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/d3e16fab48b9468b85a9c727cd9f1de0 2024-11-17T20:24:34,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/d3e16fab48b9468b85a9c727cd9f1de0, entries=150, sequenceid=273, filesize=12.0 K 2024-11-17T20:24:34,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a6e14f196acc9941f50558a2b30d8d6a in 1009ms, sequenceid=273, compaction requested=true 2024-11-17T20:24:34,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:34,650 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:34,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:34,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:34,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:34,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:34,650 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:34,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:34,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:34,652 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:34,652 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:34,652 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:34,653 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/15904128924a47719f8090ecca207369, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d9709e7a08e1447db060a0d6a1a8c694, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f0f032a8400b4e40bd2ee0a3b1e0b889, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/63fffd84bb03487b92186cb8d130aac8] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=139.0 K 2024-11-17T20:24:34,653 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:34,653 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/15904128924a47719f8090ecca207369, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d9709e7a08e1447db060a0d6a1a8c694, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f0f032a8400b4e40bd2ee0a3b1e0b889, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/63fffd84bb03487b92186cb8d130aac8] 2024-11-17T20:24:34,653 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15904128924a47719f8090ecca207369, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875069328 2024-11-17T20:24:34,653 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:34,653 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:34,654 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:34,654 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/40090ff30a20439c9adb20bda7f70a39, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/16c2fa5f8a9340e2831eb8c6fcdcab51, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/abc5dcfba0714dd6a5ade7503a8ae2c1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0a6efddf0e79425a94665f7c7e524a6e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=48.1 K 2024-11-17T20:24:34,654 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9709e7a08e1447db060a0d6a1a8c694, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731875070517 2024-11-17T20:24:34,654 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 40090ff30a20439c9adb20bda7f70a39, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875069328 2024-11-17T20:24:34,655 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f032a8400b4e40bd2ee0a3b1e0b889, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731875071176 2024-11-17T20:24:34,657 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63fffd84bb03487b92186cb8d130aac8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731875073006 2024-11-17T20:24:34,657 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c2fa5f8a9340e2831eb8c6fcdcab51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731875070517 2024-11-17T20:24:34,658 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting abc5dcfba0714dd6a5ade7503a8ae2c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731875071176 2024-11-17T20:24:34,659 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6efddf0e79425a94665f7c7e524a6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731875073007 2024-11-17T20:24:34,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T20:24:34,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:34,668 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:34,668 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:24:34,674 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:34,675 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/b82f7f619e6b47a59f0e2b5e38f5725c is 50, key is test_row_0/B:col10/1731875073014/Put/seqid=0 2024-11-17T20:24:34,681 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117f298319454e7439c94a24036d84a53b7_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:34,684 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117f298319454e7439c94a24036d84a53b7_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:34,684 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f298319454e7439c94a24036d84a53b7_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179cacf27d1c7f4a6992ee87aab4a8e898_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875073661/Put/seqid=0 2024-11-17T20:24:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742013_1189 (size=12915) 2024-11-17T20:24:34,697 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/b82f7f619e6b47a59f0e2b5e38f5725c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/b82f7f619e6b47a59f0e2b5e38f5725c 2024-11-17T20:24:34,704 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into b82f7f619e6b47a59f0e2b5e38f5725c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:34,704 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:34,704 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=12, startTime=1731875074650; duration=0sec 2024-11-17T20:24:34,704 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:34,704 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:34,704 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:34,706 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:34,706 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/C is initiating minor compaction (all files) 2024-11-17T20:24:34,706 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/C in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:34,707 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5c252dca23374902be3f36196907a039, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/568fe00fed0c4585ae3f6008b8085283, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/1d7b98462a0941a2936feddb598a6f5a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/d3e16fab48b9468b85a9c727cd9f1de0] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=48.0 K 2024-11-17T20:24:34,708 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c252dca23374902be3f36196907a039, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875069328 2024-11-17T20:24:34,709 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 568fe00fed0c4585ae3f6008b8085283, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731875070517 2024-11-17T20:24:34,709 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7b98462a0941a2936feddb598a6f5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731875071176 2024-11-17T20:24:34,710 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e16fab48b9468b85a9c727cd9f1de0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731875073007 2024-11-17T20:24:34,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742014_1190 (size=4469) 2024-11-17T20:24:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742015_1191 (size=12454) 2024-11-17T20:24:34,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:34,742 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:34,743 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/7579428a6338476094291321ca46c296 is 50, key is test_row_0/C:col10/1731875073014/Put/seqid=0 2024-11-17T20:24:34,749 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179cacf27d1c7f4a6992ee87aab4a8e898_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179cacf27d1c7f4a6992ee87aab4a8e898_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:34,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/01748918be424794ba37949cc941a710, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:34,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/01748918be424794ba37949cc941a710 is 175, key is test_row_0/A:col10/1731875073661/Put/seqid=0 2024-11-17T20:24:34,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742016_1192 (size=31255) 2024-11-17T20:24:34,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742017_1193 (size=12881) 2024-11-17T20:24:34,778 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/7579428a6338476094291321ca46c296 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7579428a6338476094291321ca46c296 2024-11-17T20:24:34,787 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/C of a6e14f196acc9941f50558a2b30d8d6a into 7579428a6338476094291321ca46c296(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:34,788 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:34,788 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/C, priority=12, startTime=1731875074650; duration=0sec 2024-11-17T20:24:34,788 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:34,788 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:34,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T20:24:34,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875134887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:34,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875134892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:34,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875134893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875135000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875135009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875135009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,121 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#155 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:35,122 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/38b6acc48cd3476c8390618d8b0189f2 is 175, key is test_row_0/A:col10/1731875073014/Put/seqid=0 2024-11-17T20:24:35,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742018_1194 (size=31869) 2024-11-17T20:24:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T20:24:35,168 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/01748918be424794ba37949cc941a710 2024-11-17T20:24:35,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/82454d2018304fafab112fdc10541669 is 50, key is test_row_0/B:col10/1731875073661/Put/seqid=0 2024-11-17T20:24:35,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742019_1195 (size=12301) 2024-11-17T20:24:35,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875135210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875135219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875135219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875135409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875135410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,410 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:35,410 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in a6e14f196acc9941f50558a2b30d8d6a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875135519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875135530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875135530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:35,534 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/38b6acc48cd3476c8390618d8b0189f2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/38b6acc48cd3476c8390618d8b0189f2 2024-11-17T20:24:35,544 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 38b6acc48cd3476c8390618d8b0189f2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:35,544 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:35,544 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=12, startTime=1731875074650; duration=0sec 2024-11-17T20:24:35,544 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:35,544 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:35,585 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/82454d2018304fafab112fdc10541669 2024-11-17T20:24:35,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5d0e46909d9a448581c59d488ecebb1e is 50, key is test_row_0/C:col10/1731875073661/Put/seqid=0 2024-11-17T20:24:35,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742020_1196 (size=12301) 2024-11-17T20:24:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T20:24:36,002 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5d0e46909d9a448581c59d488ecebb1e 2024-11-17T20:24:36,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/01748918be424794ba37949cc941a710 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/01748918be424794ba37949cc941a710 2024-11-17T20:24:36,011 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/01748918be424794ba37949cc941a710, entries=150, sequenceid=285, filesize=30.5 K 2024-11-17T20:24:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/82454d2018304fafab112fdc10541669 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/82454d2018304fafab112fdc10541669 2024-11-17T20:24:36,017 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/82454d2018304fafab112fdc10541669, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T20:24:36,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/5d0e46909d9a448581c59d488ecebb1e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5d0e46909d9a448581c59d488ecebb1e 2024-11-17T20:24:36,022 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5d0e46909d9a448581c59d488ecebb1e, entries=150, sequenceid=285, filesize=12.0 K 2024-11-17T20:24:36,023 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for a6e14f196acc9941f50558a2b30d8d6a in 1355ms, sequenceid=285, compaction requested=false 2024-11-17T20:24:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-17T20:24:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-17T20:24:36,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T20:24:36,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-11-17T20:24:36,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.5160 sec 2024-11-17T20:24:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:36,033 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:36,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T20:24:36,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179e512e87f8ab4f6bb847c66c540447a9_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875074884/Put/seqid=0 2024-11-17T20:24:36,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742021_1197 (size=14994) 2024-11-17T20:24:36,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:36,051 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179e512e87f8ab4f6bb847c66c540447a9_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179e512e87f8ab4f6bb847c66c540447a9_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:36,052 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/24461121b64242dd88bb0f014c6fda75, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:36,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875136049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/24461121b64242dd88bb0f014c6fda75 is 175, key is test_row_0/A:col10/1731875074884/Put/seqid=0 2024-11-17T20:24:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875136051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875136051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742022_1198 (size=39949) 2024-11-17T20:24:36,059 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/24461121b64242dd88bb0f014c6fda75 2024-11-17T20:24:36,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/2f9d04ffc4154e2ea3b4172c62365591 is 50, key is test_row_0/B:col10/1731875074884/Put/seqid=0 2024-11-17T20:24:36,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742023_1199 (size=12301) 2024-11-17T20:24:36,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/2f9d04ffc4154e2ea3b4172c62365591 2024-11-17T20:24:36,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/8cb4184442bd49dcb01d932c9a9c8656 is 50, key is test_row_0/C:col10/1731875074884/Put/seqid=0 2024-11-17T20:24:36,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742024_1200 (size=12301) 2024-11-17T20:24:36,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/8cb4184442bd49dcb01d932c9a9c8656 2024-11-17T20:24:36,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/24461121b64242dd88bb0f014c6fda75 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/24461121b64242dd88bb0f014c6fda75 2024-11-17T20:24:36,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/24461121b64242dd88bb0f014c6fda75, entries=200, sequenceid=314, filesize=39.0 K 2024-11-17T20:24:36,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/2f9d04ffc4154e2ea3b4172c62365591 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/2f9d04ffc4154e2ea3b4172c62365591 2024-11-17T20:24:36,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/2f9d04ffc4154e2ea3b4172c62365591, entries=150, sequenceid=314, filesize=12.0 K 2024-11-17T20:24:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/8cb4184442bd49dcb01d932c9a9c8656 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/8cb4184442bd49dcb01d932c9a9c8656 2024-11-17T20:24:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875136159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875136159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875136160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/8cb4184442bd49dcb01d932c9a9c8656, entries=150, sequenceid=314, filesize=12.0 K 2024-11-17T20:24:36,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for a6e14f196acc9941f50558a2b30d8d6a in 133ms, sequenceid=314, compaction requested=true 2024-11-17T20:24:36,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:36,167 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:36,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:36,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:36,167 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:36,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:36,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:36,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:36,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:36,168 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:36,168 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:36,168 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,169 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/38b6acc48cd3476c8390618d8b0189f2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/01748918be424794ba37949cc941a710, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/24461121b64242dd88bb0f014c6fda75] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=100.7 K 2024-11-17T20:24:36,169 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,169 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/38b6acc48cd3476c8390618d8b0189f2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/01748918be424794ba37949cc941a710, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/24461121b64242dd88bb0f014c6fda75] 2024-11-17T20:24:36,169 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:36,169 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:36,169 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,169 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/b82f7f619e6b47a59f0e2b5e38f5725c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/82454d2018304fafab112fdc10541669, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/2f9d04ffc4154e2ea3b4172c62365591] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=36.6 K 2024-11-17T20:24:36,170 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b6acc48cd3476c8390618d8b0189f2, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731875073007 2024-11-17T20:24:36,170 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting b82f7f619e6b47a59f0e2b5e38f5725c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731875073007 2024-11-17T20:24:36,171 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 82454d2018304fafab112fdc10541669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731875073654 2024-11-17T20:24:36,171 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01748918be424794ba37949cc941a710, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731875073654 2024-11-17T20:24:36,172 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9d04ffc4154e2ea3b4172c62365591, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875074884 2024-11-17T20:24:36,172 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24461121b64242dd88bb0f014c6fda75, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875074884 2024-11-17T20:24:36,183 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:36,186 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:36,187 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/31a50e958bf947d9ac9adadd730e95ee is 50, key is test_row_0/B:col10/1731875074884/Put/seqid=0 2024-11-17T20:24:36,197 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117dbfaac4fe6794c66874f49e9f18d9913_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:36,198 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117dbfaac4fe6794c66874f49e9f18d9913_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:36,198 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dbfaac4fe6794c66874f49e9f18d9913_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:36,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742026_1202 (size=4469) 2024-11-17T20:24:36,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742025_1201 (size=13017) 2024-11-17T20:24:36,212 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#164 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:36,213 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/53828ae3f3eb4f0fa41dc5db1f43c5e3 is 175, key is test_row_0/A:col10/1731875074884/Put/seqid=0 2024-11-17T20:24:36,219 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/31a50e958bf947d9ac9adadd730e95ee as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/31a50e958bf947d9ac9adadd730e95ee 2024-11-17T20:24:36,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742027_1203 (size=31971) 2024-11-17T20:24:36,229 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into 31a50e958bf947d9ac9adadd730e95ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:36,229 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:36,229 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=13, startTime=1731875076167; duration=0sec 2024-11-17T20:24:36,229 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:36,229 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:36,229 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:36,230 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:36,231 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/C is initiating minor compaction (all files) 2024-11-17T20:24:36,231 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/C in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,231 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7579428a6338476094291321ca46c296, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5d0e46909d9a448581c59d488ecebb1e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/8cb4184442bd49dcb01d932c9a9c8656] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=36.6 K 2024-11-17T20:24:36,231 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7579428a6338476094291321ca46c296, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731875073007 2024-11-17T20:24:36,232 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0e46909d9a448581c59d488ecebb1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731875073654 2024-11-17T20:24:36,232 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb4184442bd49dcb01d932c9a9c8656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875074884 2024-11-17T20:24:36,241 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#C#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:36,241 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/9b4fe12228254b8f87fd398a2a9943e4 is 50, key is test_row_0/C:col10/1731875074884/Put/seqid=0 2024-11-17T20:24:36,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742028_1204 (size=12983) 2024-11-17T20:24:36,262 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/9b4fe12228254b8f87fd398a2a9943e4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9b4fe12228254b8f87fd398a2a9943e4 2024-11-17T20:24:36,269 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/C of a6e14f196acc9941f50558a2b30d8d6a into 9b4fe12228254b8f87fd398a2a9943e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:36,270 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:36,270 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/C, priority=13, startTime=1731875076167; duration=0sec 2024-11-17T20:24:36,270 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:36,270 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:36,372 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:36,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:24:36,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a198c7dd7eca418ea113a2c3d45674e5_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875076371/Put/seqid=0 2024-11-17T20:24:36,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742029_1205 (size=14994) 2024-11-17T20:24:36,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:36,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875136413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875136414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875136414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a198c7dd7eca418ea113a2c3d45674e5_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a198c7dd7eca418ea113a2c3d45674e5_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:36,422 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/93418ae4a376424bb221d610c211fe84, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:36,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/93418ae4a376424bb221d610c211fe84 is 175, key is test_row_0/A:col10/1731875076371/Put/seqid=0 2024-11-17T20:24:36,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742030_1206 (size=39949) 2024-11-17T20:24:36,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875136520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875136529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875136529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,625 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/53828ae3f3eb4f0fa41dc5db1f43c5e3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/53828ae3f3eb4f0fa41dc5db1f43c5e3 2024-11-17T20:24:36,630 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 53828ae3f3eb4f0fa41dc5db1f43c5e3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:36,630 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:36,630 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=13, startTime=1731875076166; duration=0sec 2024-11-17T20:24:36,630 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:36,630 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T20:24:36,659 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:36,660 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-17T20:24:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T20:24:36,662 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:36,662 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:36,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:36,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875136730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875136739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875136739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T20:24:36,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T20:24:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:36,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/93418ae4a376424bb221d610c211fe84 2024-11-17T20:24:36,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/758b3e3569d8432abe3755901668aa35 is 50, key is test_row_0/B:col10/1731875076371/Put/seqid=0 2024-11-17T20:24:36,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742031_1207 (size=12301) 2024-11-17T20:24:36,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/758b3e3569d8432abe3755901668aa35 2024-11-17T20:24:36,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/c6e6800b1b194b9299994d21738b6c59 is 50, key is test_row_0/C:col10/1731875076371/Put/seqid=0 2024-11-17T20:24:36,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742032_1208 (size=12301) 2024-11-17T20:24:36,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/c6e6800b1b194b9299994d21738b6c59 2024-11-17T20:24:36,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/93418ae4a376424bb221d610c211fe84 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/93418ae4a376424bb221d610c211fe84 2024-11-17T20:24:36,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/93418ae4a376424bb221d610c211fe84, entries=200, sequenceid=329, filesize=39.0 K 2024-11-17T20:24:36,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/758b3e3569d8432abe3755901668aa35 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/758b3e3569d8432abe3755901668aa35 2024-11-17T20:24:36,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/758b3e3569d8432abe3755901668aa35, entries=150, sequenceid=329, filesize=12.0 K 2024-11-17T20:24:36,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/c6e6800b1b194b9299994d21738b6c59 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/c6e6800b1b194b9299994d21738b6c59 2024-11-17T20:24:36,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/c6e6800b1b194b9299994d21738b6c59, entries=150, sequenceid=329, filesize=12.0 K 2024-11-17T20:24:36,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a6e14f196acc9941f50558a2b30d8d6a in 541ms, sequenceid=329, compaction requested=false 2024-11-17T20:24:36,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:36,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T20:24:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:36,970 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:24:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T20:24:36,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a7fc756cd2844d489a6ce3b1a9b533f3_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875076413/Put/seqid=0 2024-11-17T20:24:36,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742033_1209 (size=12454) 2024-11-17T20:24:37,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:37,011 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a7fc756cd2844d489a6ce3b1a9b533f3_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a7fc756cd2844d489a6ce3b1a9b533f3_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/624fcf0b6fd94a8b9ff84c6c09c3579c, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:37,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/624fcf0b6fd94a8b9ff84c6c09c3579c is 175, key is test_row_0/A:col10/1731875076413/Put/seqid=0 2024-11-17T20:24:37,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742034_1210 (size=31255) 2024-11-17T20:24:37,024 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/624fcf0b6fd94a8b9ff84c6c09c3579c 2024-11-17T20:24:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:37,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:37,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6cce6109f278470c91623ed08ac4cb05 is 50, key is test_row_0/B:col10/1731875076413/Put/seqid=0 2024-11-17T20:24:37,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742035_1211 (size=12301) 2024-11-17T20:24:37,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875137069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875137069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875137070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,078 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6cce6109f278470c91623ed08ac4cb05 2024-11-17T20:24:37,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/9ea37af6a00d4974913a128e5f0ebbae is 50, key is test_row_0/C:col10/1731875076413/Put/seqid=0 2024-11-17T20:24:37,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742036_1212 (size=12301) 2024-11-17T20:24:37,106 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/9ea37af6a00d4974913a128e5f0ebbae 2024-11-17T20:24:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/624fcf0b6fd94a8b9ff84c6c09c3579c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/624fcf0b6fd94a8b9ff84c6c09c3579c 2024-11-17T20:24:37,125 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/624fcf0b6fd94a8b9ff84c6c09c3579c, entries=150, sequenceid=354, filesize=30.5 K 2024-11-17T20:24:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/6cce6109f278470c91623ed08ac4cb05 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6cce6109f278470c91623ed08ac4cb05 2024-11-17T20:24:37,131 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6cce6109f278470c91623ed08ac4cb05, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T20:24:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/9ea37af6a00d4974913a128e5f0ebbae as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9ea37af6a00d4974913a128e5f0ebbae 2024-11-17T20:24:37,139 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9ea37af6a00d4974913a128e5f0ebbae, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T20:24:37,141 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for a6e14f196acc9941f50558a2b30d8d6a in 171ms, sequenceid=354, compaction requested=true 2024-11-17T20:24:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-17T20:24:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-17T20:24:37,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T20:24:37,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 481 msec 2024-11-17T20:24:37,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 486 msec 2024-11-17T20:24:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:37,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:37,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:24:37,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176c76c4fd44de4ee2bbe21a738edc209c_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875077182/Put/seqid=0 2024-11-17T20:24:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742037_1213 (size=12454) 2024-11-17T20:24:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875137216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875137218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875137222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T20:24:37,290 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:37,290 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-17T20:24:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T20:24:37,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:37,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:37,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:37,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875137332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875137333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875137333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T20:24:37,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:37,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:37,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875137540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875137540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875137549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,610 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T20:24:37,614 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176c76c4fd44de4ee2bbe21a738edc209c_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176c76c4fd44de4ee2bbe21a738edc209c_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:37,616 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/49aa6336eb0d44f5a6b47bc984c3ca54, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:37,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/49aa6336eb0d44f5a6b47bc984c3ca54 is 175, key is test_row_0/A:col10/1731875077182/Put/seqid=0 2024-11-17T20:24:37,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742038_1214 (size=31255) 2024-11-17T20:24:37,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:37,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:37,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875137850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875137850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875137859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:37,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:37,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:37,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:37,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T20:24:38,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/49aa6336eb0d44f5a6b47bc984c3ca54 2024-11-17T20:24:38,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/64ac54b9f8a54e74926ca1c3943a391c is 50, key is test_row_0/B:col10/1731875077182/Put/seqid=0 2024-11-17T20:24:38,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:38,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:38,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742039_1215 (size=12301) 2024-11-17T20:24:38,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/64ac54b9f8a54e74926ca1c3943a391c 2024-11-17T20:24:38,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/6d94c492e77146ccb985505b27801e57 is 50, key is test_row_0/C:col10/1731875077182/Put/seqid=0 2024-11-17T20:24:38,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742040_1216 (size=12301) 2024-11-17T20:24:38,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:38,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:38,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875138359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:38,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875138359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:38,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:38,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875138369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T20:24:38,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/6d94c492e77146ccb985505b27801e57 2024-11-17T20:24:38,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/49aa6336eb0d44f5a6b47bc984c3ca54 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/49aa6336eb0d44f5a6b47bc984c3ca54 2024-11-17T20:24:38,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/49aa6336eb0d44f5a6b47bc984c3ca54, entries=150, sequenceid=370, filesize=30.5 K 2024-11-17T20:24:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/64ac54b9f8a54e74926ca1c3943a391c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/64ac54b9f8a54e74926ca1c3943a391c 2024-11-17T20:24:38,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/64ac54b9f8a54e74926ca1c3943a391c, entries=150, sequenceid=370, filesize=12.0 K 2024-11-17T20:24:38,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/6d94c492e77146ccb985505b27801e57 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/6d94c492e77146ccb985505b27801e57 2024-11-17T20:24:38,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/6d94c492e77146ccb985505b27801e57, entries=150, sequenceid=370, filesize=12.0 K 2024-11-17T20:24:38,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a6e14f196acc9941f50558a2b30d8d6a in 1318ms, sequenceid=370, compaction requested=true 2024-11-17T20:24:38,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:38,501 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:38,501 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:38,502 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:38,502 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:38,502 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,502 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/31a50e958bf947d9ac9adadd730e95ee, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/758b3e3569d8432abe3755901668aa35, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6cce6109f278470c91623ed08ac4cb05, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/64ac54b9f8a54e74926ca1c3943a391c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=48.8 K 2024-11-17T20:24:38,503 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:38,503 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:38,503 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,503 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/53828ae3f3eb4f0fa41dc5db1f43c5e3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/93418ae4a376424bb221d610c211fe84, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/624fcf0b6fd94a8b9ff84c6c09c3579c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/49aa6336eb0d44f5a6b47bc984c3ca54] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=131.3 K 2024-11-17T20:24:38,503 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,503 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/53828ae3f3eb4f0fa41dc5db1f43c5e3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/93418ae4a376424bb221d610c211fe84, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/624fcf0b6fd94a8b9ff84c6c09c3579c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/49aa6336eb0d44f5a6b47bc984c3ca54] 2024-11-17T20:24:38,503 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a50e958bf947d9ac9adadd730e95ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875074884 2024-11-17T20:24:38,504 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 758b3e3569d8432abe3755901668aa35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731875076042 2024-11-17T20:24:38,504 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53828ae3f3eb4f0fa41dc5db1f43c5e3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875074884 2024-11-17T20:24:38,504 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cce6109f278470c91623ed08ac4cb05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731875076404 2024-11-17T20:24:38,504 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93418ae4a376424bb221d610c211fe84, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731875076042 2024-11-17T20:24:38,504 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ac54b9f8a54e74926ca1c3943a391c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731875077067 2024-11-17T20:24:38,504 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624fcf0b6fd94a8b9ff84c6c09c3579c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731875076404 2024-11-17T20:24:38,505 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49aa6336eb0d44f5a6b47bc984c3ca54, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731875077067 2024-11-17T20:24:38,513 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:38,514 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:38,515 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ad36b6c0e76b44d9a513399ac9e7b0c1 is 50, key is test_row_0/B:col10/1731875077182/Put/seqid=0 2024-11-17T20:24:38,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-17T20:24:38,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,518 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T20:24:38,524 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ce4b0f81a5bf477483f191009e696581_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:38,526 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ce4b0f81a5bf477483f191009e696581_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:38,526 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ce4b0f81a5bf477483f191009e696581_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:38,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742041_1217 (size=13153) 2024-11-17T20:24:38,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171cb4eb2245fe4847b1bfa64ee557f59a_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875077221/Put/seqid=0 2024-11-17T20:24:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742042_1218 (size=4469) 2024-11-17T20:24:38,558 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#176 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:38,559 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/6b3c6abd473e4493b0840a57659718a1 is 175, key is test_row_0/A:col10/1731875077182/Put/seqid=0 2024-11-17T20:24:38,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742043_1219 (size=12454) 2024-11-17T20:24:38,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742044_1220 (size=32107) 2024-11-17T20:24:38,952 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ad36b6c0e76b44d9a513399ac9e7b0c1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ad36b6c0e76b44d9a513399ac9e7b0c1 2024-11-17T20:24:38,965 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into ad36b6c0e76b44d9a513399ac9e7b0c1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:38,965 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:38,965 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=12, startTime=1731875078501; duration=0sec 2024-11-17T20:24:38,965 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:38,965 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:38,965 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:38,971 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:38,971 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/C is initiating minor compaction (all files) 2024-11-17T20:24:38,971 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/C in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:38,972 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9b4fe12228254b8f87fd398a2a9943e4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/c6e6800b1b194b9299994d21738b6c59, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9ea37af6a00d4974913a128e5f0ebbae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/6d94c492e77146ccb985505b27801e57] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=48.7 K 2024-11-17T20:24:38,972 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b4fe12228254b8f87fd398a2a9943e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875074884 2024-11-17T20:24:38,973 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e6800b1b194b9299994d21738b6c59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731875076042 2024-11-17T20:24:38,973 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea37af6a00d4974913a128e5f0ebbae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731875076404 2024-11-17T20:24:38,973 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d94c492e77146ccb985505b27801e57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731875077067 2024-11-17T20:24:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:38,985 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171cb4eb2245fe4847b1bfa64ee557f59a_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171cb4eb2245fe4847b1bfa64ee557f59a_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/aa7112124cad47b5bfcfaa4d194b342e, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/aa7112124cad47b5bfcfaa4d194b342e is 175, key is test_row_0/A:col10/1731875077221/Put/seqid=0 2024-11-17T20:24:39,003 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/6b3c6abd473e4493b0840a57659718a1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/6b3c6abd473e4493b0840a57659718a1 2024-11-17T20:24:39,003 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:39,003 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/62ab8248814e4bccb8b2bf41f88bfd38 is 50, key is test_row_0/C:col10/1731875077182/Put/seqid=0 2024-11-17T20:24:39,011 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 6b3c6abd473e4493b0840a57659718a1(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:39,011 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:39,011 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=12, startTime=1731875078500; duration=0sec 2024-11-17T20:24:39,012 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:39,012 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:39,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742045_1221 (size=31255) 2024-11-17T20:24:39,015 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/aa7112124cad47b5bfcfaa4d194b342e 2024-11-17T20:24:39,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/d13bf6ac231a443198106f5e45139230 is 50, key is test_row_0/B:col10/1731875077221/Put/seqid=0 2024-11-17T20:24:39,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742046_1222 (size=13119) 2024-11-17T20:24:39,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742047_1223 (size=12301) 2024-11-17T20:24:39,045 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/d13bf6ac231a443198106f5e45139230 2024-11-17T20:24:39,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/7ee47dabe2c343cfb65a22487e02d352 is 50, key is test_row_0/C:col10/1731875077221/Put/seqid=0 2024-11-17T20:24:39,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742048_1224 (size=12301) 2024-11-17T20:24:39,062 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/7ee47dabe2c343cfb65a22487e02d352 2024-11-17T20:24:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/aa7112124cad47b5bfcfaa4d194b342e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/aa7112124cad47b5bfcfaa4d194b342e 2024-11-17T20:24:39,070 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/aa7112124cad47b5bfcfaa4d194b342e, entries=150, sequenceid=390, filesize=30.5 K 2024-11-17T20:24:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/d13bf6ac231a443198106f5e45139230 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d13bf6ac231a443198106f5e45139230 2024-11-17T20:24:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,082 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d13bf6ac231a443198106f5e45139230, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T20:24:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/7ee47dabe2c343cfb65a22487e02d352 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7ee47dabe2c343cfb65a22487e02d352 2024-11-17T20:24:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,092 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7ee47dabe2c343cfb65a22487e02d352, entries=150, sequenceid=390, filesize=12.0 K 2024-11-17T20:24:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,093 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for a6e14f196acc9941f50558a2b30d8d6a in 576ms, sequenceid=390, compaction requested=false 2024-11-17T20:24:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:39,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:39,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-17T20:24:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-17T20:24:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-17T20:24:39,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-17T20:24:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.8070 sec 2024-11-17T20:24:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,417 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:39,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:24:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e4c071daeadd48a0a3c612d554fc1e20_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875079378/Put/seqid=0 2024-11-17T20:24:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T20:24:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,440 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:39,440 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/62ab8248814e4bccb8b2bf41f88bfd38 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/62ab8248814e4bccb8b2bf41f88bfd38 2024-11-17T20:24:39,441 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-17T20:24:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T20:24:39,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,445 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:39,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,450 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/C of a6e14f196acc9941f50558a2b30d8d6a into 62ab8248814e4bccb8b2bf41f88bfd38(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,450 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:39,450 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/C, priority=12, startTime=1731875078501; duration=0sec 2024-11-17T20:24:39,450 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:39,450 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742050_1226 (size=25158) 2024-11-17T20:24:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875139468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875139466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875139469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875139470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875139472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T20:24:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875139579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875139579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875139579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875139579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875139590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:39,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T20:24:39,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:39,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875139790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875139790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875139790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875139790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875139799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:39,858 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:39,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e4c071daeadd48a0a3c612d554fc1e20_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e4c071daeadd48a0a3c612d554fc1e20_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:39,869 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fb15854af132422d8a26c9188012734c, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:39,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fb15854af132422d8a26c9188012734c is 175, key is test_row_0/A:col10/1731875079378/Put/seqid=0 2024-11-17T20:24:39,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742049_1225 (size=74794) 2024-11-17T20:24:39,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fb15854af132422d8a26c9188012734c 2024-11-17T20:24:39,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/46a1576ecf284cc6b24c934b5a5b2310 is 50, key is test_row_0/B:col10/1731875079378/Put/seqid=0 2024-11-17T20:24:39,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742051_1227 (size=12301) 2024-11-17T20:24:39,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:39,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:39,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:39,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:39,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T20:24:40,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875140099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875140100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875140100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875140100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875140109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/46a1576ecf284cc6b24c934b5a5b2310 2024-11-17T20:24:40,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/94f2e14227f74668a82b0e82b20cc09c is 50, key is test_row_0/C:col10/1731875079378/Put/seqid=0 2024-11-17T20:24:40,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742052_1228 (size=12301) 2024-11-17T20:24:40,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T20:24:40,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731875140610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731875140610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60444 deadline: 1731875140610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1731875140610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60404 deadline: 1731875140619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:24:40,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/94f2e14227f74668a82b0e82b20cc09c 2024-11-17T20:24:40,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:40,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:40,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/fb15854af132422d8a26c9188012734c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fb15854af132422d8a26c9188012734c 2024-11-17T20:24:40,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fb15854af132422d8a26c9188012734c, entries=400, sequenceid=403, filesize=73.0 K 2024-11-17T20:24:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/46a1576ecf284cc6b24c934b5a5b2310 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/46a1576ecf284cc6b24c934b5a5b2310 2024-11-17T20:24:40,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/46a1576ecf284cc6b24c934b5a5b2310, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T20:24:40,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/94f2e14227f74668a82b0e82b20cc09c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/94f2e14227f74668a82b0e82b20cc09c 2024-11-17T20:24:40,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/94f2e14227f74668a82b0e82b20cc09c, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T20:24:40,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a6e14f196acc9941f50558a2b30d8d6a in 1319ms, sequenceid=403, compaction requested=true 2024-11-17T20:24:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T20:24:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6e14f196acc9941f50558a2b30d8d6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-17T20:24:40,737 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:40,737 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:40,739 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:40,739 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:40,739 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/A is initiating minor compaction (all files) 2024-11-17T20:24:40,739 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/C is initiating minor compaction (all files) 2024-11-17T20:24:40,739 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/A in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,739 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/C in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,740 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/6b3c6abd473e4493b0840a57659718a1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/aa7112124cad47b5bfcfaa4d194b342e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fb15854af132422d8a26c9188012734c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=134.9 K 2024-11-17T20:24:40,740 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/62ab8248814e4bccb8b2bf41f88bfd38, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7ee47dabe2c343cfb65a22487e02d352, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/94f2e14227f74668a82b0e82b20cc09c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=36.8 K 2024-11-17T20:24:40,740 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,740 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/6b3c6abd473e4493b0840a57659718a1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/aa7112124cad47b5bfcfaa4d194b342e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fb15854af132422d8a26c9188012734c] 2024-11-17T20:24:40,740 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ab8248814e4bccb8b2bf41f88bfd38, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731875077067 2024-11-17T20:24:40,740 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3c6abd473e4493b0840a57659718a1, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731875077067 2024-11-17T20:24:40,741 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ee47dabe2c343cfb65a22487e02d352, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731875077211 2024-11-17T20:24:40,741 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa7112124cad47b5bfcfaa4d194b342e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731875077211 2024-11-17T20:24:40,741 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb15854af132422d8a26c9188012734c, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731875079378 2024-11-17T20:24:40,741 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f2e14227f74668a82b0e82b20cc09c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731875079378 2024-11-17T20:24:40,749 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:40,753 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#C#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:40,754 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/ee3a64aaa6934b92b743400d77618c47 is 50, key is test_row_0/C:col10/1731875079378/Put/seqid=0 2024-11-17T20:24:40,777 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117fe1f9fe830eb40118aaa9d80f843e23b_a6e14f196acc9941f50558a2b30d8d6a store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:40,779 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117fe1f9fe830eb40118aaa9d80f843e23b_a6e14f196acc9941f50558a2b30d8d6a, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:40,779 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fe1f9fe830eb40118aaa9d80f843e23b_a6e14f196acc9941f50558a2b30d8d6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742054_1230 (size=4469) 2024-11-17T20:24:40,805 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#A#compaction#185 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:40,806 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/921ff118dd5346878b5702a41ae2493f is 175, key is test_row_0/A:col10/1731875079378/Put/seqid=0 2024-11-17T20:24:40,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742053_1229 (size=13221) 2024-11-17T20:24:40,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742055_1231 (size=32209) 2024-11-17T20:24:40,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-17T20:24:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:40,860 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:24:40,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117872c86f82abf42d3af47eb9bf5e98d3a_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875079467/Put/seqid=0 2024-11-17T20:24:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742056_1232 (size=12454) 2024-11-17T20:24:41,216 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/ee3a64aaa6934b92b743400d77618c47 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/ee3a64aaa6934b92b743400d77618c47 2024-11-17T20:24:41,220 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/C of a6e14f196acc9941f50558a2b30d8d6a into ee3a64aaa6934b92b743400d77618c47(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:41,220 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:41,221 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/C, priority=13, startTime=1731875080737; duration=0sec 2024-11-17T20:24:41,221 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:41,221 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:C 2024-11-17T20:24:41,221 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:41,222 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:41,222 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): a6e14f196acc9941f50558a2b30d8d6a/B is initiating minor compaction (all files) 2024-11-17T20:24:41,222 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6e14f196acc9941f50558a2b30d8d6a/B in TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:41,222 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ad36b6c0e76b44d9a513399ac9e7b0c1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d13bf6ac231a443198106f5e45139230, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/46a1576ecf284cc6b24c934b5a5b2310] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp, totalSize=36.9 K 2024-11-17T20:24:41,223 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ad36b6c0e76b44d9a513399ac9e7b0c1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731875077067 2024-11-17T20:24:41,223 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d13bf6ac231a443198106f5e45139230, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731875077211 2024-11-17T20:24:41,224 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a1576ecf284cc6b24c934b5a5b2310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731875079378 2024-11-17T20:24:41,227 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/921ff118dd5346878b5702a41ae2493f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/921ff118dd5346878b5702a41ae2493f 2024-11-17T20:24:41,231 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6e14f196acc9941f50558a2b30d8d6a#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:41,232 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/92e4923794a344b2905d8d530138662a is 50, key is test_row_0/B:col10/1731875079378/Put/seqid=0 2024-11-17T20:24:41,232 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/A of a6e14f196acc9941f50558a2b30d8d6a into 921ff118dd5346878b5702a41ae2493f(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:41,232 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:41,232 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/A, priority=13, startTime=1731875080737; duration=0sec 2024-11-17T20:24:41,232 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:41,232 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:A 2024-11-17T20:24:41,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742057_1233 (size=13255) 2024-11-17T20:24:41,241 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/92e4923794a344b2905d8d530138662a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/92e4923794a344b2905d8d530138662a 2024-11-17T20:24:41,248 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6e14f196acc9941f50558a2b30d8d6a/B of a6e14f196acc9941f50558a2b30d8d6a into 92e4923794a344b2905d8d530138662a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:41,248 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:41,248 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., storeName=a6e14f196acc9941f50558a2b30d8d6a/B, priority=13, startTime=1731875080737; duration=0sec 2024-11-17T20:24:41,248 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:41,248 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6e14f196acc9941f50558a2b30d8d6a:B 2024-11-17T20:24:41,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:41,293 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117872c86f82abf42d3af47eb9bf5e98d3a_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117872c86f82abf42d3af47eb9bf5e98d3a_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:41,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/0d734da672ef43cd991aacd0e9d94253, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:41,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/0d734da672ef43cd991aacd0e9d94253 is 175, key is test_row_0/A:col10/1731875079467/Put/seqid=0 2024-11-17T20:24:41,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742058_1234 (size=31255) 2024-11-17T20:24:41,317 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/0d734da672ef43cd991aacd0e9d94253 2024-11-17T20:24:41,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ac4fe1b5f31645c39bbaf76be854eb14 is 50, key is test_row_0/B:col10/1731875079467/Put/seqid=0 2024-11-17T20:24:41,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742059_1235 (size=12301) 2024-11-17T20:24:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T20:24:41,594 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-11-17T20:24:41,594 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,594 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,594 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-11-17T20:24:41,594 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,594 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,594 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,594 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,595 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-17T20:24:41,595 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,595 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,595 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,598 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-11-17T20:24:41,598 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,598 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,598 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:41,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. as already flushing 2024-11-17T20:24:41,622 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-17T20:24:41,622 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-17T20:24:41,622 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-17T20:24:41,622 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,622 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,622 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,622 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,622 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,622 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,623 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,623 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,623 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,631 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-17T20:24:41,631 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-11-17T20:24:41,632 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,632 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:24:41,632 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,632 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,632 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,632 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:41,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:41,732 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ac4fe1b5f31645c39bbaf76be854eb14 2024-11-17T20:24:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/27e7f21577ee41b1b1f08d218aa22a9c is 50, key is test_row_0/C:col10/1731875079467/Put/seqid=0 2024-11-17T20:24:41,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742060_1236 (size=12301) 2024-11-17T20:24:42,149 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/27e7f21577ee41b1b1f08d218aa22a9c 2024-11-17T20:24:42,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/0d734da672ef43cd991aacd0e9d94253 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/0d734da672ef43cd991aacd0e9d94253 2024-11-17T20:24:42,163 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/0d734da672ef43cd991aacd0e9d94253, entries=150, sequenceid=429, filesize=30.5 K 2024-11-17T20:24:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/ac4fe1b5f31645c39bbaf76be854eb14 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ac4fe1b5f31645c39bbaf76be854eb14 2024-11-17T20:24:42,171 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ac4fe1b5f31645c39bbaf76be854eb14, entries=150, sequenceid=429, filesize=12.0 K 2024-11-17T20:24:42,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/27e7f21577ee41b1b1f08d218aa22a9c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/27e7f21577ee41b1b1f08d218aa22a9c 2024-11-17T20:24:42,176 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/27e7f21577ee41b1b1f08d218aa22a9c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-17T20:24:42,177 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for a6e14f196acc9941f50558a2b30d8d6a in 1317ms, sequenceid=429, compaction requested=false 2024-11-17T20:24:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for a6e14f196acc9941f50558a2b30d8d6a: 2024-11-17T20:24:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-17T20:24:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-17T20:24:42,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T20:24:42,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7330 sec 2024-11-17T20:24:42,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.7400 sec 2024-11-17T20:24:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T20:24:43,602 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3291 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3175 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1422 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4266 rows 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1417 2024-11-17T20:24:43,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4248 rows 2024-11-17T20:24:43,603 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:24:43,603 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:24:43,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:43,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:43,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:24:43,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T20:24:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T20:24:43,613 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875083613"}]},"ts":"1731875083613"} 2024-11-17T20:24:43,616 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T20:24:43,616 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T20:24:43,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:24:43,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, UNASSIGN}] 2024-11-17T20:24:43,619 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, UNASSIGN 2024-11-17T20:24:43,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:43,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, UNASSIGN because future has completed 2024-11-17T20:24:43,622 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:24:43,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:24:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T20:24:43,776 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:43,776 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:24:43,777 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing a6e14f196acc9941f50558a2b30d8d6a, disabling compactions & flushes 2024-11-17T20:24:43,777 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:43,777 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:43,777 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. after waiting 0 ms 2024-11-17T20:24:43,777 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:43,777 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing a6e14f196acc9941f50558a2b30d8d6a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T20:24:43,788 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c00520b0da5f43e6823c386dc4ce2919_a6e14f196acc9941f50558a2b30d8d6a is 50, key is test_row_0/A:col10/1731875081630/Put/seqid=0 2024-11-17T20:24:43,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742061_1237 (size=9914) 2024-11-17T20:24:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T20:24:44,195 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:44,204 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c00520b0da5f43e6823c386dc4ce2919_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c00520b0da5f43e6823c386dc4ce2919_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:44,205 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7939b1e8b7944ddd9c44e01edcc5596c, store: [table=TestAcidGuarantees family=A region=a6e14f196acc9941f50558a2b30d8d6a] 2024-11-17T20:24:44,206 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7939b1e8b7944ddd9c44e01edcc5596c is 175, key is test_row_0/A:col10/1731875081630/Put/seqid=0 2024-11-17T20:24:44,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742062_1238 (size=22561) 2024-11-17T20:24:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T20:24:44,611 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7939b1e8b7944ddd9c44e01edcc5596c 2024-11-17T20:24:44,679 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0f6a3b264a134cfb8ff8e409484b415a is 50, key is test_row_0/B:col10/1731875081630/Put/seqid=0 2024-11-17T20:24:44,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742063_1239 (size=9857) 2024-11-17T20:24:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T20:24:45,088 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0f6a3b264a134cfb8ff8e409484b415a 2024-11-17T20:24:45,102 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/08b336dc4e664abaa00ee02a19ce66b5 is 50, key is test_row_0/C:col10/1731875081630/Put/seqid=0 2024-11-17T20:24:45,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742064_1240 (size=9857) 2024-11-17T20:24:45,506 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/08b336dc4e664abaa00ee02a19ce66b5 2024-11-17T20:24:45,510 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/A/7939b1e8b7944ddd9c44e01edcc5596c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7939b1e8b7944ddd9c44e01edcc5596c 2024-11-17T20:24:45,513 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7939b1e8b7944ddd9c44e01edcc5596c, entries=100, sequenceid=440, filesize=22.0 K 2024-11-17T20:24:45,514 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/B/0f6a3b264a134cfb8ff8e409484b415a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0f6a3b264a134cfb8ff8e409484b415a 2024-11-17T20:24:45,517 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0f6a3b264a134cfb8ff8e409484b415a, entries=100, sequenceid=440, filesize=9.6 K 2024-11-17T20:24:45,518 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/.tmp/C/08b336dc4e664abaa00ee02a19ce66b5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/08b336dc4e664abaa00ee02a19ce66b5 2024-11-17T20:24:45,522 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/08b336dc4e664abaa00ee02a19ce66b5, entries=100, sequenceid=440, filesize=9.6 K 2024-11-17T20:24:45,523 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a6e14f196acc9941f50558a2b30d8d6a in 1746ms, sequenceid=440, compaction requested=true 2024-11-17T20:24:45,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7b5f7f3a43bc44a6a6bf44cb3077da43, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4efb1aed431a4cdaabc94281e6684ad5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/224ea6bbc748472ca4762bd42c85d84c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fcfa6eadee224ed5b4f5132550f03976, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/28d4f70966b4468fb052791ca354847b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/e7e10117506c4f8c8c29a3d840fa41f6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/36a3f4017e0141768779efb63b4ecf80, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f44b819f3c1c4a54ac766a749d200567, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/3d6d4debe1b245c4ac5d97082d11089d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/a2ecf3f6ad0149e5a4d2f9893001db63, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4077886b070d4395a354ec6191cbc9c0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/71e929ea336c4878aa7358d817f12933, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/ea69a1edb716466790745782d7ff9d7a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d16f980f605c47d682f5b8f842952f3b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/15904128924a47719f8090ecca207369, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d9709e7a08e1447db060a0d6a1a8c694, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f0f032a8400b4e40bd2ee0a3b1e0b889, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/63fffd84bb03487b92186cb8d130aac8, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/38b6acc48cd3476c8390618d8b0189f2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/01748918be424794ba37949cc941a710, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/24461121b64242dd88bb0f014c6fda75, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/53828ae3f3eb4f0fa41dc5db1f43c5e3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/93418ae4a376424bb221d610c211fe84, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/624fcf0b6fd94a8b9ff84c6c09c3579c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/6b3c6abd473e4493b0840a57659718a1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/49aa6336eb0d44f5a6b47bc984c3ca54, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/aa7112124cad47b5bfcfaa4d194b342e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fb15854af132422d8a26c9188012734c] to archive 2024-11-17T20:24:45,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:24:45,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7b5f7f3a43bc44a6a6bf44cb3077da43 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7b5f7f3a43bc44a6a6bf44cb3077da43 2024-11-17T20:24:45,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4efb1aed431a4cdaabc94281e6684ad5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4efb1aed431a4cdaabc94281e6684ad5 2024-11-17T20:24:45,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/224ea6bbc748472ca4762bd42c85d84c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/224ea6bbc748472ca4762bd42c85d84c 2024-11-17T20:24:45,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fcfa6eadee224ed5b4f5132550f03976 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fcfa6eadee224ed5b4f5132550f03976 2024-11-17T20:24:45,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/28d4f70966b4468fb052791ca354847b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/28d4f70966b4468fb052791ca354847b 2024-11-17T20:24:45,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/e7e10117506c4f8c8c29a3d840fa41f6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/e7e10117506c4f8c8c29a3d840fa41f6 2024-11-17T20:24:45,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/36a3f4017e0141768779efb63b4ecf80 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/36a3f4017e0141768779efb63b4ecf80 2024-11-17T20:24:45,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f44b819f3c1c4a54ac766a749d200567 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f44b819f3c1c4a54ac766a749d200567 2024-11-17T20:24:45,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/3d6d4debe1b245c4ac5d97082d11089d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/3d6d4debe1b245c4ac5d97082d11089d 2024-11-17T20:24:45,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/a2ecf3f6ad0149e5a4d2f9893001db63 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/a2ecf3f6ad0149e5a4d2f9893001db63 2024-11-17T20:24:45,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4077886b070d4395a354ec6191cbc9c0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/4077886b070d4395a354ec6191cbc9c0 2024-11-17T20:24:45,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/71e929ea336c4878aa7358d817f12933 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/71e929ea336c4878aa7358d817f12933 2024-11-17T20:24:45,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/ea69a1edb716466790745782d7ff9d7a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/ea69a1edb716466790745782d7ff9d7a 2024-11-17T20:24:45,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d16f980f605c47d682f5b8f842952f3b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d16f980f605c47d682f5b8f842952f3b 2024-11-17T20:24:45,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/15904128924a47719f8090ecca207369 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/15904128924a47719f8090ecca207369 2024-11-17T20:24:45,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d9709e7a08e1447db060a0d6a1a8c694 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/d9709e7a08e1447db060a0d6a1a8c694 2024-11-17T20:24:45,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f0f032a8400b4e40bd2ee0a3b1e0b889 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/f0f032a8400b4e40bd2ee0a3b1e0b889 2024-11-17T20:24:45,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/63fffd84bb03487b92186cb8d130aac8 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/63fffd84bb03487b92186cb8d130aac8 2024-11-17T20:24:45,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/38b6acc48cd3476c8390618d8b0189f2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/38b6acc48cd3476c8390618d8b0189f2 2024-11-17T20:24:45,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/01748918be424794ba37949cc941a710 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/01748918be424794ba37949cc941a710 2024-11-17T20:24:45,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/24461121b64242dd88bb0f014c6fda75 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/24461121b64242dd88bb0f014c6fda75 2024-11-17T20:24:45,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/53828ae3f3eb4f0fa41dc5db1f43c5e3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/53828ae3f3eb4f0fa41dc5db1f43c5e3 2024-11-17T20:24:45,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/93418ae4a376424bb221d610c211fe84 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/93418ae4a376424bb221d610c211fe84 2024-11-17T20:24:45,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/624fcf0b6fd94a8b9ff84c6c09c3579c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/624fcf0b6fd94a8b9ff84c6c09c3579c 2024-11-17T20:24:45,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/6b3c6abd473e4493b0840a57659718a1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/6b3c6abd473e4493b0840a57659718a1 2024-11-17T20:24:45,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/49aa6336eb0d44f5a6b47bc984c3ca54 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/49aa6336eb0d44f5a6b47bc984c3ca54 2024-11-17T20:24:45,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/aa7112124cad47b5bfcfaa4d194b342e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/aa7112124cad47b5bfcfaa4d194b342e 2024-11-17T20:24:45,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fb15854af132422d8a26c9188012734c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/fb15854af132422d8a26c9188012734c 2024-11-17T20:24:45,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ca911ca52061410090825e3a7a3b8cf2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4b5a7d34f10845ddaea550bada0b8224, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ff80358b39df4201b04e3280431ad956, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/9e17c76f5f86458caf6d6995a6f47d51, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/a4c9b3eec14d41fcb2906449f64c9da4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/55fcc2892ca14e6db14c6f008c60ba2c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/17fdbda036df4175b5107d45c969cec9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d3e45b22fb1a455e90705d3210676d00, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/26e9ff9b1a134357b1a0a7a3f31a1f7b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4df57ab963304ce388ac7fce72e630e3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e099493873a46c492d88b4cc7d5bcb9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e662f43a6524eb4931b08e2e3fb1907, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/c27d2a94c4464accbf18d5ea2b62ebfd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/40090ff30a20439c9adb20bda7f70a39, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0ac3f310e98e4583a37734eb5659492d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/16c2fa5f8a9340e2831eb8c6fcdcab51, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/abc5dcfba0714dd6a5ade7503a8ae2c1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/b82f7f619e6b47a59f0e2b5e38f5725c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0a6efddf0e79425a94665f7c7e524a6e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/82454d2018304fafab112fdc10541669, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/31a50e958bf947d9ac9adadd730e95ee, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/2f9d04ffc4154e2ea3b4172c62365591, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/758b3e3569d8432abe3755901668aa35, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6cce6109f278470c91623ed08ac4cb05, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ad36b6c0e76b44d9a513399ac9e7b0c1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/64ac54b9f8a54e74926ca1c3943a391c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d13bf6ac231a443198106f5e45139230, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/46a1576ecf284cc6b24c934b5a5b2310] to archive 2024-11-17T20:24:45,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:24:45,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ca911ca52061410090825e3a7a3b8cf2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ca911ca52061410090825e3a7a3b8cf2 2024-11-17T20:24:45,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4b5a7d34f10845ddaea550bada0b8224 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4b5a7d34f10845ddaea550bada0b8224 2024-11-17T20:24:45,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ff80358b39df4201b04e3280431ad956 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ff80358b39df4201b04e3280431ad956 2024-11-17T20:24:45,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/9e17c76f5f86458caf6d6995a6f47d51 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/9e17c76f5f86458caf6d6995a6f47d51 2024-11-17T20:24:45,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/a4c9b3eec14d41fcb2906449f64c9da4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/a4c9b3eec14d41fcb2906449f64c9da4 2024-11-17T20:24:45,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/55fcc2892ca14e6db14c6f008c60ba2c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/55fcc2892ca14e6db14c6f008c60ba2c 2024-11-17T20:24:45,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/17fdbda036df4175b5107d45c969cec9 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/17fdbda036df4175b5107d45c969cec9 2024-11-17T20:24:45,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d3e45b22fb1a455e90705d3210676d00 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d3e45b22fb1a455e90705d3210676d00 2024-11-17T20:24:45,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/26e9ff9b1a134357b1a0a7a3f31a1f7b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/26e9ff9b1a134357b1a0a7a3f31a1f7b 2024-11-17T20:24:45,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4df57ab963304ce388ac7fce72e630e3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/4df57ab963304ce388ac7fce72e630e3 2024-11-17T20:24:45,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e099493873a46c492d88b4cc7d5bcb9 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e099493873a46c492d88b4cc7d5bcb9 2024-11-17T20:24:45,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e662f43a6524eb4931b08e2e3fb1907 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6e662f43a6524eb4931b08e2e3fb1907 2024-11-17T20:24:45,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/c27d2a94c4464accbf18d5ea2b62ebfd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/c27d2a94c4464accbf18d5ea2b62ebfd 2024-11-17T20:24:45,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/40090ff30a20439c9adb20bda7f70a39 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/40090ff30a20439c9adb20bda7f70a39 2024-11-17T20:24:45,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0ac3f310e98e4583a37734eb5659492d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0ac3f310e98e4583a37734eb5659492d 2024-11-17T20:24:45,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/16c2fa5f8a9340e2831eb8c6fcdcab51 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/16c2fa5f8a9340e2831eb8c6fcdcab51 2024-11-17T20:24:45,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/abc5dcfba0714dd6a5ade7503a8ae2c1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/abc5dcfba0714dd6a5ade7503a8ae2c1 2024-11-17T20:24:45,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/b82f7f619e6b47a59f0e2b5e38f5725c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/b82f7f619e6b47a59f0e2b5e38f5725c 2024-11-17T20:24:45,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0a6efddf0e79425a94665f7c7e524a6e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0a6efddf0e79425a94665f7c7e524a6e 2024-11-17T20:24:45,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/82454d2018304fafab112fdc10541669 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/82454d2018304fafab112fdc10541669 2024-11-17T20:24:45,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/31a50e958bf947d9ac9adadd730e95ee to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/31a50e958bf947d9ac9adadd730e95ee 2024-11-17T20:24:45,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/2f9d04ffc4154e2ea3b4172c62365591 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/2f9d04ffc4154e2ea3b4172c62365591 2024-11-17T20:24:45,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/758b3e3569d8432abe3755901668aa35 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/758b3e3569d8432abe3755901668aa35 2024-11-17T20:24:45,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6cce6109f278470c91623ed08ac4cb05 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/6cce6109f278470c91623ed08ac4cb05 2024-11-17T20:24:45,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ad36b6c0e76b44d9a513399ac9e7b0c1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ad36b6c0e76b44d9a513399ac9e7b0c1 2024-11-17T20:24:45,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/64ac54b9f8a54e74926ca1c3943a391c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/64ac54b9f8a54e74926ca1c3943a391c 2024-11-17T20:24:45,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d13bf6ac231a443198106f5e45139230 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/d13bf6ac231a443198106f5e45139230 2024-11-17T20:24:45,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/46a1576ecf284cc6b24c934b5a5b2310 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/46a1576ecf284cc6b24c934b5a5b2310 2024-11-17T20:24:45,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f8af53f690b544038f58e08b64f3e177, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2876939cfb5845a88bbbfd7ee58df316, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/74b1954533a24a15aa4be25fe04f194b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/66ba0e7cccab4806a95db561c60de397, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7eeef9d742634183901d2f4a5dac2e01, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f27c3bcdd35f45b883f6c9c679710e46, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/11f85a588a4b4957920a8df932f9db4a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5e8ebaf65ac34a3dae13b5462e4fe23d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/4fa5177985f64b11ade9e619aaf99bdb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2efb58b513344cd38e4ad28f96a9bec4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/42b540e8517046799f7c72bdc7f01506, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/34d8b12dbf174ff3a069717673bf0684, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5c252dca23374902be3f36196907a039, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/625aea588cfb492ea07b1bf4ee085129, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/568fe00fed0c4585ae3f6008b8085283, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/1d7b98462a0941a2936feddb598a6f5a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7579428a6338476094291321ca46c296, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/d3e16fab48b9468b85a9c727cd9f1de0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5d0e46909d9a448581c59d488ecebb1e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9b4fe12228254b8f87fd398a2a9943e4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/8cb4184442bd49dcb01d932c9a9c8656, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/c6e6800b1b194b9299994d21738b6c59, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9ea37af6a00d4974913a128e5f0ebbae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/62ab8248814e4bccb8b2bf41f88bfd38, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/6d94c492e77146ccb985505b27801e57, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7ee47dabe2c343cfb65a22487e02d352, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/94f2e14227f74668a82b0e82b20cc09c] to archive 2024-11-17T20:24:45,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:24:45,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f8af53f690b544038f58e08b64f3e177 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f8af53f690b544038f58e08b64f3e177 2024-11-17T20:24:45,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2876939cfb5845a88bbbfd7ee58df316 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2876939cfb5845a88bbbfd7ee58df316 2024-11-17T20:24:45,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/74b1954533a24a15aa4be25fe04f194b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/74b1954533a24a15aa4be25fe04f194b 2024-11-17T20:24:45,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/66ba0e7cccab4806a95db561c60de397 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/66ba0e7cccab4806a95db561c60de397 2024-11-17T20:24:45,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7eeef9d742634183901d2f4a5dac2e01 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7eeef9d742634183901d2f4a5dac2e01 2024-11-17T20:24:45,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f27c3bcdd35f45b883f6c9c679710e46 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/f27c3bcdd35f45b883f6c9c679710e46 2024-11-17T20:24:45,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/11f85a588a4b4957920a8df932f9db4a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/11f85a588a4b4957920a8df932f9db4a 2024-11-17T20:24:45,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5e8ebaf65ac34a3dae13b5462e4fe23d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5e8ebaf65ac34a3dae13b5462e4fe23d 2024-11-17T20:24:45,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/4fa5177985f64b11ade9e619aaf99bdb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/4fa5177985f64b11ade9e619aaf99bdb 2024-11-17T20:24:45,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2efb58b513344cd38e4ad28f96a9bec4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/2efb58b513344cd38e4ad28f96a9bec4 2024-11-17T20:24:45,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/42b540e8517046799f7c72bdc7f01506 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/42b540e8517046799f7c72bdc7f01506 2024-11-17T20:24:45,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/34d8b12dbf174ff3a069717673bf0684 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/34d8b12dbf174ff3a069717673bf0684 2024-11-17T20:24:45,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5c252dca23374902be3f36196907a039 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5c252dca23374902be3f36196907a039 2024-11-17T20:24:45,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/625aea588cfb492ea07b1bf4ee085129 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/625aea588cfb492ea07b1bf4ee085129 2024-11-17T20:24:45,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/568fe00fed0c4585ae3f6008b8085283 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/568fe00fed0c4585ae3f6008b8085283 2024-11-17T20:24:45,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/1d7b98462a0941a2936feddb598a6f5a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/1d7b98462a0941a2936feddb598a6f5a 2024-11-17T20:24:45,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7579428a6338476094291321ca46c296 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7579428a6338476094291321ca46c296 2024-11-17T20:24:45,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/d3e16fab48b9468b85a9c727cd9f1de0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/d3e16fab48b9468b85a9c727cd9f1de0 2024-11-17T20:24:45,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5d0e46909d9a448581c59d488ecebb1e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/5d0e46909d9a448581c59d488ecebb1e 2024-11-17T20:24:45,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9b4fe12228254b8f87fd398a2a9943e4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9b4fe12228254b8f87fd398a2a9943e4 2024-11-17T20:24:45,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/8cb4184442bd49dcb01d932c9a9c8656 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/8cb4184442bd49dcb01d932c9a9c8656 2024-11-17T20:24:45,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/c6e6800b1b194b9299994d21738b6c59 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/c6e6800b1b194b9299994d21738b6c59 2024-11-17T20:24:45,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9ea37af6a00d4974913a128e5f0ebbae to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/9ea37af6a00d4974913a128e5f0ebbae 2024-11-17T20:24:45,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/62ab8248814e4bccb8b2bf41f88bfd38 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/62ab8248814e4bccb8b2bf41f88bfd38 2024-11-17T20:24:45,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/6d94c492e77146ccb985505b27801e57 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/6d94c492e77146ccb985505b27801e57 2024-11-17T20:24:45,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7ee47dabe2c343cfb65a22487e02d352 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/7ee47dabe2c343cfb65a22487e02d352 2024-11-17T20:24:45,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/94f2e14227f74668a82b0e82b20cc09c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/94f2e14227f74668a82b0e82b20cc09c 2024-11-17T20:24:45,631 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/recovered.edits/443.seqid, newMaxSeqId=443, maxSeqId=4 2024-11-17T20:24:45,632 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. 2024-11-17T20:24:45,632 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for a6e14f196acc9941f50558a2b30d8d6a: Waiting for close lock at 1731875083777Running coprocessor pre-close hooks at 1731875083777Disabling compacts and flushes for region at 1731875083777Disabling writes for close at 1731875083777Obtaining lock to block concurrent updates at 1731875083777Preparing flush snapshotting stores in a6e14f196acc9941f50558a2b30d8d6a at 1731875083777Finished memstore snapshotting TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731875083778 (+1 ms)Flushing stores of TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a. at 1731875083779 (+1 ms)Flushing a6e14f196acc9941f50558a2b30d8d6a/A: creating writer at 1731875083779Flushing mob file a6e14f196acc9941f50558a2b30d8d6a/A: appending metadata at 1731875083788 (+9 ms)Flushing mob file a6e14f196acc9941f50558a2b30d8d6a/A: closing flushed file at 1731875083788Flushing a6e14f196acc9941f50558a2b30d8d6a/A: appending metadata at 1731875084205 (+417 ms)Flushing a6e14f196acc9941f50558a2b30d8d6a/A: closing flushed file at 1731875084205Flushing a6e14f196acc9941f50558a2b30d8d6a/B: creating writer at 1731875084672 (+467 ms)Flushing a6e14f196acc9941f50558a2b30d8d6a/B: appending metadata at 1731875084678 (+6 ms)Flushing a6e14f196acc9941f50558a2b30d8d6a/B: closing flushed file at 1731875084678Flushing a6e14f196acc9941f50558a2b30d8d6a/C: creating writer at 1731875085099 (+421 ms)Flushing a6e14f196acc9941f50558a2b30d8d6a/C: appending metadata at 1731875085102 (+3 ms)Flushing a6e14f196acc9941f50558a2b30d8d6a/C: closing flushed file at 1731875085102Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@509a1a49: reopening flushed file at 1731875085510 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c6264: reopening flushed file at 1731875085514 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31b587f9: reopening flushed file at 1731875085517 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a6e14f196acc9941f50558a2b30d8d6a in 1746ms, sequenceid=440, compaction requested=true at 1731875085523 (+6 ms)Writing region close event to WAL at 1731875085628 (+105 ms)Running coprocessor post-close hooks at 1731875085632 (+4 ms)Closed at 1731875085632 2024-11-17T20:24:45,634 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,635 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=a6e14f196acc9941f50558a2b30d8d6a, regionState=CLOSED 2024-11-17T20:24:45,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:24:45,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-17T20:24:45,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure a6e14f196acc9941f50558a2b30d8d6a, server=0b0b0ae8de03,46555,1731875029029 in 2.0140 sec 2024-11-17T20:24:45,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-17T20:24:45,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6e14f196acc9941f50558a2b30d8d6a, UNASSIGN in 2.0210 sec 2024-11-17T20:24:45,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T20:24:45,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0250 sec 2024-11-17T20:24:45,645 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875085644"}]},"ts":"1731875085644"} 2024-11-17T20:24:45,647 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T20:24:45,647 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T20:24:45,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0420 sec 2024-11-17T20:24:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T20:24:45,760 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:45,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T20:24:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:45,763 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T20:24:45,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:45,769 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,773 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/recovered.edits] 2024-11-17T20:24:45,777 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/0d734da672ef43cd991aacd0e9d94253 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/0d734da672ef43cd991aacd0e9d94253 2024-11-17T20:24:45,779 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7939b1e8b7944ddd9c44e01edcc5596c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/7939b1e8b7944ddd9c44e01edcc5596c 2024-11-17T20:24:45,780 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/921ff118dd5346878b5702a41ae2493f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/A/921ff118dd5346878b5702a41ae2493f 2024-11-17T20:24:45,783 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0f6a3b264a134cfb8ff8e409484b415a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/0f6a3b264a134cfb8ff8e409484b415a 2024-11-17T20:24:45,784 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/92e4923794a344b2905d8d530138662a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/92e4923794a344b2905d8d530138662a 2024-11-17T20:24:45,785 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ac4fe1b5f31645c39bbaf76be854eb14 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/B/ac4fe1b5f31645c39bbaf76be854eb14 2024-11-17T20:24:45,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/08b336dc4e664abaa00ee02a19ce66b5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/08b336dc4e664abaa00ee02a19ce66b5 2024-11-17T20:24:45,789 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/27e7f21577ee41b1b1f08d218aa22a9c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/27e7f21577ee41b1b1f08d218aa22a9c 2024-11-17T20:24:45,789 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/ee3a64aaa6934b92b743400d77618c47 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/C/ee3a64aaa6934b92b743400d77618c47 2024-11-17T20:24:45,792 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/recovered.edits/443.seqid to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a/recovered.edits/443.seqid 2024-11-17T20:24:45,792 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,792 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T20:24:45,793 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T20:24:45,793 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T20:24:45,797 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709c987a3695e4afd868a9a8fea9ae079_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709c987a3695e4afd868a9a8fea9ae079_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,798 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170d7e65d79a69407294cfc7ce959b99e9_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170d7e65d79a69407294cfc7ce959b99e9_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,799 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171cb4eb2245fe4847b1bfa64ee557f59a_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171cb4eb2245fe4847b1bfa64ee557f59a_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,800 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174a7f8b0d74e04191a59b6e8512beeaa6_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174a7f8b0d74e04191a59b6e8512beeaa6_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,801 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b31da27a2514fe7b477731b2ba23dc8_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b31da27a2514fe7b477731b2ba23dc8_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,802 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ad22da1cad64737b970d467bab1abb2_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ad22da1cad64737b970d467bab1abb2_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,803 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176b86648e26d344a9a6446c7febe74256_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176b86648e26d344a9a6446c7febe74256_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,804 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176c76c4fd44de4ee2bbe21a738edc209c_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176c76c4fd44de4ee2bbe21a738edc209c_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,805 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117754c3c62251145e6ad1ad7010ae06e2b_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117754c3c62251145e6ad1ad7010ae06e2b_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,806 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117872c86f82abf42d3af47eb9bf5e98d3a_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117872c86f82abf42d3af47eb9bf5e98d3a_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,807 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111787a8bb5a5cc446dabc975a424d1c09cc_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111787a8bb5a5cc446dabc975a424d1c09cc_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,808 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179cacf27d1c7f4a6992ee87aab4a8e898_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179cacf27d1c7f4a6992ee87aab4a8e898_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,810 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179e512e87f8ab4f6bb847c66c540447a9_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179e512e87f8ab4f6bb847c66c540447a9_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,811 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a198c7dd7eca418ea113a2c3d45674e5_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a198c7dd7eca418ea113a2c3d45674e5_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,812 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a7fc756cd2844d489a6ce3b1a9b533f3_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a7fc756cd2844d489a6ce3b1a9b533f3_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,813 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae6770e4d4984b85bc23a191d65d1acb_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae6770e4d4984b85bc23a191d65d1acb_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,814 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7acccb528364ac8ba9c0bcf0e79e95a_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7acccb528364ac8ba9c0bcf0e79e95a_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,815 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c00520b0da5f43e6823c386dc4ce2919_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c00520b0da5f43e6823c386dc4ce2919_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,815 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c5412c0046f7469295abf66dfdbb057e_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c5412c0046f7469295abf66dfdbb057e_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,816 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dc189406e0a14325a65859fcaed068d6_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dc189406e0a14325a65859fcaed068d6_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,817 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ddc0dcc809b54ff6a99832127daa9a33_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ddc0dcc809b54ff6a99832127daa9a33_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,818 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e223cfe4df9945a399a6dc8ead00224f_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e223cfe4df9945a399a6dc8ead00224f_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,819 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e4c071daeadd48a0a3c612d554fc1e20_a6e14f196acc9941f50558a2b30d8d6a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e4c071daeadd48a0a3c612d554fc1e20_a6e14f196acc9941f50558a2b30d8d6a 2024-11-17T20:24:45,819 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T20:24:45,821 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:45,823 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T20:24:45,824 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T20:24:45,825 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:45,825 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T20:24:45,826 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731875085825"}]},"ts":"9223372036854775807"} 2024-11-17T20:24:45,828 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T20:24:45,828 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a6e14f196acc9941f50558a2b30d8d6a, NAME => 'TestAcidGuarantees,,1731875058411.a6e14f196acc9941f50558a2b30d8d6a.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T20:24:45,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T20:24:45,828 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731875085828"}]},"ts":"9223372036854775807"} 2024-11-17T20:24:45,830 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T20:24:45,831 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:45,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-17T20:24:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T20:24:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T20:24:45,869 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:45,881 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=245 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1718410319_22 at /127.0.0.1:38142 [Waiting for operation #976] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-941759243_22 at /127.0.0.1:37816 [Waiting for operation #1067] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-941759243_22 at /127.0.0.1:38158 [Waiting for operation #978] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1718410319_22 at /127.0.0.1:60264 [Waiting for operation #1312] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=499 (was 474) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8101 (was 8562) 2024-11-17T20:24:45,890 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=245, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=499, ProcessCount=11, AvailableMemoryMB=8101 2024-11-17T20:24:45,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:24:45,892 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:24:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T20:24:45,894 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T20:24:45,894 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:45,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-17T20:24:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-17T20:24:45,895 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T20:24:45,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742065_1241 (size=959) 2024-11-17T20:24:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-17T20:24:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-17T20:24:46,306 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:24:46,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742066_1242 (size=53) 2024-11-17T20:24:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-17T20:24:46,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:24:46,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 31e1f74c01abc6e3eb193ea5bbd4a22a, disabling compactions & flushes 2024-11-17T20:24:46,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:46,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:46,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. after waiting 0 ms 2024-11-17T20:24:46,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:46,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:46,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: Waiting for close lock at 1731875086718Disabling compacts and flushes for region at 1731875086718Disabling writes for close at 1731875086718Writing region close event to WAL at 1731875086718Closed at 1731875086718 2024-11-17T20:24:46,721 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T20:24:46,721 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731875086721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875086721"}]},"ts":"1731875086721"} 2024-11-17T20:24:46,724 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T20:24:46,725 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T20:24:46,726 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875086725"}]},"ts":"1731875086725"} 2024-11-17T20:24:46,727 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T20:24:46,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, ASSIGN}] 2024-11-17T20:24:46,729 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, ASSIGN 2024-11-17T20:24:46,730 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=false 2024-11-17T20:24:46,881 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=31e1f74c01abc6e3eb193ea5bbd4a22a, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:46,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, ASSIGN because future has completed 2024-11-17T20:24:46,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:24:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-17T20:24:47,045 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:47,045 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:24:47,045 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,046 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:24:47,046 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,046 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,047 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,049 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31e1f74c01abc6e3eb193ea5bbd4a22a columnFamilyName A 2024-11-17T20:24:47,049 DEBUG [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:47,049 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] regionserver.HStore(327): Store=31e1f74c01abc6e3eb193ea5bbd4a22a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:47,049 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,051 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31e1f74c01abc6e3eb193ea5bbd4a22a columnFamilyName B 2024-11-17T20:24:47,051 DEBUG [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:47,051 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] regionserver.HStore(327): Store=31e1f74c01abc6e3eb193ea5bbd4a22a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:47,052 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,053 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31e1f74c01abc6e3eb193ea5bbd4a22a columnFamilyName C 2024-11-17T20:24:47,053 DEBUG [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:24:47,053 INFO [StoreOpener-31e1f74c01abc6e3eb193ea5bbd4a22a-1 {}] regionserver.HStore(327): Store=31e1f74c01abc6e3eb193ea5bbd4a22a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:24:47,054 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,054 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,055 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,056 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,056 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,057 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:24:47,058 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,060 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:24:47,061 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 31e1f74c01abc6e3eb193ea5bbd4a22a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73684242, jitterRate=0.09798076748847961}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:24:47,061 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:47,062 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: Running coprocessor pre-open hook at 1731875087046Writing region info on filesystem at 1731875087046Initializing all the Stores at 1731875087047 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875087047Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875087047Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875087047Cleaning up temporary data from old regions at 1731875087056 (+9 ms)Running coprocessor post-open hooks at 1731875087061 (+5 ms)Region opened successfully at 1731875087062 (+1 ms) 2024-11-17T20:24:47,062 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., pid=69, masterSystemTime=1731875087041 2024-11-17T20:24:47,064 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:47,064 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:47,065 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=31e1f74c01abc6e3eb193ea5bbd4a22a, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:47,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:24:47,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T20:24:47,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 in 181 msec 2024-11-17T20:24:47,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-17T20:24:47,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, ASSIGN in 341 msec 2024-11-17T20:24:47,071 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T20:24:47,071 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875087071"}]},"ts":"1731875087071"} 2024-11-17T20:24:47,073 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T20:24:47,074 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T20:24:47,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-17T20:24:47,390 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T20:24:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-17T20:24:48,050 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:48,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,054 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d00840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,057 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c454316, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,059 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,063 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T20:24:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,32801,1731875028288 2024-11-17T20:24:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bcff41b 2024-11-17T20:24:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T20:24:48,065 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T20:24:48,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,068 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3be13557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,070 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2a60f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,072 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c7804f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,073 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1feadee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,075 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a04c65b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ada739c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6687ccb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,081 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45428bb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,083 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d2e0e5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,085 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ffc2ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,088 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f4b71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,089 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d973f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,091 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c294ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@534ec2a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,095 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd404b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,096 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50cf040d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,097 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c877f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7592f3cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:24:48,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:24:48,100 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:24:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:24:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f613b66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:24:48,101 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:24:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1872b5b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:24:48,102 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,102 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,102 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,102 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,103 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,103 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,104 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,104 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,105 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T20:24:48,105 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,107 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T20:24:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,107 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:24:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,109 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:24:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:24:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:24:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:24:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:48,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:48,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:24:48,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/692365f3ee2d44ba91fc4efc148604fd is 50, key is test_row_0/A:col10/1731875088116/Put/seqid=0 2024-11-17T20:24:48,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875148144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875148147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875148150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875148150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875148152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742067_1243 (size=12001) 2024-11-17T20:24:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T20:24:48,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875148259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875148259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875148259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875148259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875148259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:48,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:48,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T20:24:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875148469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875148469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875148469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875148470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875148470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/692365f3ee2d44ba91fc4efc148604fd 2024-11-17T20:24:48,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/0f9c3432d6dd4ef1aaa06510b3e9a34e is 50, key is test_row_0/B:col10/1731875088116/Put/seqid=0 2024-11-17T20:24:48,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742068_1244 (size=12001) 2024-11-17T20:24:48,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T20:24:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875148779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875148779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875148779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875148779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875148790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:48,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:48,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/0f9c3432d6dd4ef1aaa06510b3e9a34e 2024-11-17T20:24:49,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c298a232958c42608fa4c228724b38cd is 50, key is test_row_0/C:col10/1731875088116/Put/seqid=0 2024-11-17T20:24:49,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742069_1245 (size=12001) 2024-11-17T20:24:49,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:49,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T20:24:49,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875149289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875149290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875149290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875149290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875149299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:49,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c298a232958c42608fa4c228724b38cd 2024-11-17T20:24:49,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/692365f3ee2d44ba91fc4efc148604fd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/692365f3ee2d44ba91fc4efc148604fd 2024-11-17T20:24:49,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/692365f3ee2d44ba91fc4efc148604fd, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T20:24:49,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/0f9c3432d6dd4ef1aaa06510b3e9a34e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/0f9c3432d6dd4ef1aaa06510b3e9a34e 2024-11-17T20:24:49,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/0f9c3432d6dd4ef1aaa06510b3e9a34e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T20:24:49,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c298a232958c42608fa4c228724b38cd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c298a232958c42608fa4c228724b38cd 2024-11-17T20:24:49,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c298a232958c42608fa4c228724b38cd, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T20:24:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:49,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1374ms, sequenceid=14, compaction requested=false 2024-11-17T20:24:49,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:49,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T20:24:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:49,645 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:24:49,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ba522a46bd64467ca9ed2706fd0f3f32 is 50, key is test_row_0/A:col10/1731875088147/Put/seqid=0 2024-11-17T20:24:49,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742070_1246 (size=12001) 2024-11-17T20:24:49,706 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ba522a46bd64467ca9ed2706fd0f3f32 2024-11-17T20:24:49,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/c006b73842ae4042ade63c0bbce1f084 is 50, key is test_row_0/B:col10/1731875088147/Put/seqid=0 2024-11-17T20:24:49,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742071_1247 (size=12001) 2024-11-17T20:24:50,123 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/c006b73842ae4042ade63c0bbce1f084 2024-11-17T20:24:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/85921f50ef9745cda4c9dd4503dbee3b is 50, key is test_row_0/C:col10/1731875088147/Put/seqid=0 2024-11-17T20:24:50,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742072_1248 (size=12001) 2024-11-17T20:24:50,137 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/85921f50ef9745cda4c9dd4503dbee3b 2024-11-17T20:24:50,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ba522a46bd64467ca9ed2706fd0f3f32 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ba522a46bd64467ca9ed2706fd0f3f32 2024-11-17T20:24:50,146 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ba522a46bd64467ca9ed2706fd0f3f32, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T20:24:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/c006b73842ae4042ade63c0bbce1f084 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/c006b73842ae4042ade63c0bbce1f084 2024-11-17T20:24:50,152 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/c006b73842ae4042ade63c0bbce1f084, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T20:24:50,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/85921f50ef9745cda4c9dd4503dbee3b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85921f50ef9745cda4c9dd4503dbee3b 2024-11-17T20:24:50,158 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85921f50ef9745cda4c9dd4503dbee3b, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T20:24:50,159 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 514ms, sequenceid=38, compaction requested=false 2024-11-17T20:24:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T20:24:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T20:24:50,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T20:24:50,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0520 sec 2024-11-17T20:24:50,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0610 sec 2024-11-17T20:24:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T20:24:50,250 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:50,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T20:24:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T20:24:50,252 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:50,253 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:50,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:50,313 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:50,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T20:24:50,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ed86c5ceeab642a0b03b304b2c387a13 is 50, key is test_row_0/A:col10/1731875090313/Put/seqid=0 2024-11-17T20:24:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875150336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742073_1249 (size=14341) 2024-11-17T20:24:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875150336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875150337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ed86c5ceeab642a0b03b304b2c387a13 2024-11-17T20:24:50,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875150342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875150343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7d6d3c7987d84adfa7806e15fe676c9e is 50, key is test_row_0/B:col10/1731875090313/Put/seqid=0 2024-11-17T20:24:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T20:24:50,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742074_1250 (size=12001) 2024-11-17T20:24:50,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T20:24:50,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:50,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875150449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875150449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875150449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875150449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875150459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T20:24:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T20:24:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875150659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875150659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875150660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875150660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875150669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T20:24:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7d6d3c7987d84adfa7806e15fe676c9e 2024-11-17T20:24:50,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/11141fd2053b464d972f3786cf9dc505 is 50, key is test_row_0/C:col10/1731875090313/Put/seqid=0 2024-11-17T20:24:50,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742075_1251 (size=12001) 2024-11-17T20:24:50,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T20:24:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:50,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T20:24:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875150969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875150969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875150969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875150969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:50,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875150979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T20:24:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:51,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:51,157 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T20:24:51,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T20:24:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:51,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:51,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:51,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/11141fd2053b464d972f3786cf9dc505 2024-11-17T20:24:51,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ed86c5ceeab642a0b03b304b2c387a13 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed86c5ceeab642a0b03b304b2c387a13 2024-11-17T20:24:51,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed86c5ceeab642a0b03b304b2c387a13, entries=200, sequenceid=52, filesize=14.0 K 2024-11-17T20:24:51,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7d6d3c7987d84adfa7806e15fe676c9e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7d6d3c7987d84adfa7806e15fe676c9e 2024-11-17T20:24:51,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7d6d3c7987d84adfa7806e15fe676c9e, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T20:24:51,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/11141fd2053b464d972f3786cf9dc505 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/11141fd2053b464d972f3786cf9dc505 2024-11-17T20:24:51,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/11141fd2053b464d972f3786cf9dc505, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T20:24:51,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 890ms, sequenceid=52, compaction requested=true 2024-11-17T20:24:51,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:51,204 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:51,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:51,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:51,204 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:51,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:51,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:51,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:51,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:51,206 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:51,206 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:24:51,206 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,206 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/692365f3ee2d44ba91fc4efc148604fd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ba522a46bd64467ca9ed2706fd0f3f32, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed86c5ceeab642a0b03b304b2c387a13] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=37.4 K 2024-11-17T20:24:51,206 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:51,206 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:24:51,207 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,207 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/0f9c3432d6dd4ef1aaa06510b3e9a34e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/c006b73842ae4042ade63c0bbce1f084, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7d6d3c7987d84adfa7806e15fe676c9e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.2 K 2024-11-17T20:24:51,207 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692365f3ee2d44ba91fc4efc148604fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731875088116 2024-11-17T20:24:51,207 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f9c3432d6dd4ef1aaa06510b3e9a34e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731875088116 2024-11-17T20:24:51,207 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba522a46bd64467ca9ed2706fd0f3f32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731875088147 2024-11-17T20:24:51,208 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c006b73842ae4042ade63c0bbce1f084, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731875088147 2024-11-17T20:24:51,208 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed86c5ceeab642a0b03b304b2c387a13, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875090302 2024-11-17T20:24:51,208 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6d3c7987d84adfa7806e15fe676c9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875090302 2024-11-17T20:24:51,230 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:51,233 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/d6702ac017c64cdfb4576a916f6da5ad is 50, key is test_row_0/B:col10/1731875090313/Put/seqid=0 2024-11-17T20:24:51,238 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:51,240 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ed1e4a1331be442d89bca112f26c54db is 50, key is test_row_0/A:col10/1731875090313/Put/seqid=0 2024-11-17T20:24:51,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742076_1252 (size=12104) 2024-11-17T20:24:51,275 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/d6702ac017c64cdfb4576a916f6da5ad as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d6702ac017c64cdfb4576a916f6da5ad 2024-11-17T20:24:51,291 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into d6702ac017c64cdfb4576a916f6da5ad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:51,291 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:51,291 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875091204; duration=0sec 2024-11-17T20:24:51,291 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:51,291 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:24:51,291 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:51,293 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:51,293 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:24:51,293 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742077_1253 (size=12104) 2024-11-17T20:24:51,293 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c298a232958c42608fa4c228724b38cd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85921f50ef9745cda4c9dd4503dbee3b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/11141fd2053b464d972f3786cf9dc505] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.2 K 2024-11-17T20:24:51,294 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c298a232958c42608fa4c228724b38cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731875088116 2024-11-17T20:24:51,294 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 85921f50ef9745cda4c9dd4503dbee3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731875088147 2024-11-17T20:24:51,295 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 11141fd2053b464d972f3786cf9dc505, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875090302 2024-11-17T20:24:51,300 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ed1e4a1331be442d89bca112f26c54db as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed1e4a1331be442d89bca112f26c54db 2024-11-17T20:24:51,306 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into ed1e4a1331be442d89bca112f26c54db(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:51,306 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:51,306 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875091204; duration=0sec 2024-11-17T20:24:51,306 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:51,306 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:24:51,307 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#205 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:51,308 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/1148a2de4bdd4a39963a753ffbcab5f0 is 50, key is test_row_0/C:col10/1731875090313/Put/seqid=0 2024-11-17T20:24:51,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T20:24:51,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,326 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T20:24:51,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742078_1254 (size=12104) 2024-11-17T20:24:51,358 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/1148a2de4bdd4a39963a753ffbcab5f0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1148a2de4bdd4a39963a753ffbcab5f0 2024-11-17T20:24:51,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a18d268fec4d4d309b7a8f1cfe5c12b0 is 50, key is test_row_0/A:col10/1731875090333/Put/seqid=0 2024-11-17T20:24:51,364 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into 1148a2de4bdd4a39963a753ffbcab5f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:51,364 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:51,364 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=13, startTime=1731875091204; duration=0sec 2024-11-17T20:24:51,364 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:51,364 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:24:51,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742079_1255 (size=12001) 2024-11-17T20:24:51,370 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a18d268fec4d4d309b7a8f1cfe5c12b0 2024-11-17T20:24:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T20:24:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/1ac63d9bf1dc41c0ab449c42b10eb33a is 50, key is test_row_0/B:col10/1731875090333/Put/seqid=0 2024-11-17T20:24:51,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742080_1256 (size=12001) 2024-11-17T20:24:51,440 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/1ac63d9bf1dc41c0ab449c42b10eb33a 2024-11-17T20:24:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c26feba91acd40129c2bd4d891472389 is 50, key is test_row_0/C:col10/1731875090333/Put/seqid=0 2024-11-17T20:24:51,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742081_1257 (size=12001) 2024-11-17T20:24:51,468 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c26feba91acd40129c2bd4d891472389 2024-11-17T20:24:51,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a18d268fec4d4d309b7a8f1cfe5c12b0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a18d268fec4d4d309b7a8f1cfe5c12b0 2024-11-17T20:24:51,479 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a18d268fec4d4d309b7a8f1cfe5c12b0, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T20:24:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/1ac63d9bf1dc41c0ab449c42b10eb33a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/1ac63d9bf1dc41c0ab449c42b10eb33a 2024-11-17T20:24:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:51,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:51,484 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/1ac63d9bf1dc41c0ab449c42b10eb33a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T20:24:51,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c26feba91acd40129c2bd4d891472389 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c26feba91acd40129c2bd4d891472389 2024-11-17T20:24:51,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875151489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875151490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875151490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,494 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c26feba91acd40129c2bd4d891472389, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T20:24:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,495 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 169ms, sequenceid=77, compaction requested=false 2024-11-17T20:24:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T20:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T20:24:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:51,508 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:51,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-17T20:24:51,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T20:24:51,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2430 sec 2024-11-17T20:24:51,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2610 sec 2024-11-17T20:24:51,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/32f6474d4c0a4baf9f2a3baa29d29d70 is 50, key is test_row_0/A:col10/1731875091498/Put/seqid=0 2024-11-17T20:24:51,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742082_1258 (size=12001) 2024-11-17T20:24:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875151579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875151580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875151599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875151600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875151600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875151689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875151689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875151809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875151809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875151810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875151899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875151899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:51,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/32f6474d4c0a4baf9f2a3baa29d29d70 2024-11-17T20:24:51,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/50c9445169764d9585819246295c13b3 is 50, key is test_row_0/B:col10/1731875091498/Put/seqid=0 2024-11-17T20:24:51,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742083_1259 (size=12001) 2024-11-17T20:24:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875152119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875152119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875152120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875152209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875152210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/50c9445169764d9585819246295c13b3 2024-11-17T20:24:52,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/137d3fdc6f7548f7882cafdd0d552385 is 50, key is test_row_0/C:col10/1731875091498/Put/seqid=0 2024-11-17T20:24:52,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742084_1260 (size=12001) 2024-11-17T20:24:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T20:24:52,399 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:52,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T20:24:52,402 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T20:24:52,403 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:52,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T20:24:52,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T20:24:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:52,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:52,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875152630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875152630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875152630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T20:24:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T20:24:52,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875152719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875152719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:52,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/137d3fdc6f7548f7882cafdd0d552385 2024-11-17T20:24:52,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/32f6474d4c0a4baf9f2a3baa29d29d70 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32f6474d4c0a4baf9f2a3baa29d29d70 2024-11-17T20:24:52,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32f6474d4c0a4baf9f2a3baa29d29d70, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T20:24:52,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/50c9445169764d9585819246295c13b3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/50c9445169764d9585819246295c13b3 2024-11-17T20:24:52,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/50c9445169764d9585819246295c13b3, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T20:24:52,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/137d3fdc6f7548f7882cafdd0d552385 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/137d3fdc6f7548f7882cafdd0d552385 2024-11-17T20:24:52,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/137d3fdc6f7548f7882cafdd0d552385, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T20:24:52,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1298ms, sequenceid=97, compaction requested=true 2024-11-17T20:24:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:52,807 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:52,807 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:52,808 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:52,808 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:24:52,808 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,808 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed1e4a1331be442d89bca112f26c54db, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a18d268fec4d4d309b7a8f1cfe5c12b0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32f6474d4c0a4baf9f2a3baa29d29d70] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.3 K 2024-11-17T20:24:52,809 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1e4a1331be442d89bca112f26c54db, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875090302 2024-11-17T20:24:52,810 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18d268fec4d4d309b7a8f1cfe5c12b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875090333 2024-11-17T20:24:52,810 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:52,810 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:24:52,810 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,810 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d6702ac017c64cdfb4576a916f6da5ad, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/1ac63d9bf1dc41c0ab449c42b10eb33a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/50c9445169764d9585819246295c13b3] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.3 K 2024-11-17T20:24:52,810 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f6474d4c0a4baf9f2a3baa29d29d70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731875091498 2024-11-17T20:24:52,810 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d6702ac017c64cdfb4576a916f6da5ad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875090302 2024-11-17T20:24:52,811 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac63d9bf1dc41c0ab449c42b10eb33a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875090333 2024-11-17T20:24:52,812 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 50c9445169764d9585819246295c13b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731875091498 2024-11-17T20:24:52,826 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:52,827 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/8b8dbe4c9f1b4152b07b73406a5743d8 is 50, key is test_row_0/A:col10/1731875091498/Put/seqid=0 2024-11-17T20:24:52,842 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:52,843 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bd267cdc7cd84121a8b6c2eac13257ae is 50, key is test_row_0/B:col10/1731875091498/Put/seqid=0 2024-11-17T20:24:52,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742085_1261 (size=12207) 2024-11-17T20:24:52,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T20:24:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,861 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/8b8dbe4c9f1b4152b07b73406a5743d8 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/8b8dbe4c9f1b4152b07b73406a5743d8 2024-11-17T20:24:52,862 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-17T20:24:52,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/1efcc9176eff447f81ed305a636677f3 is 50, key is test_row_0/A:col10/1731875091574/Put/seqid=0 2024-11-17T20:24:52,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742086_1262 (size=12207) 2024-11-17T20:24:52,869 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into 8b8dbe4c9f1b4152b07b73406a5743d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:52,869 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:52,869 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875092807; duration=0sec 2024-11-17T20:24:52,869 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:52,869 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:24:52,869 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:52,870 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:52,870 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:24:52,870 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,870 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1148a2de4bdd4a39963a753ffbcab5f0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c26feba91acd40129c2bd4d891472389, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/137d3fdc6f7548f7882cafdd0d552385] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.3 K 2024-11-17T20:24:52,871 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1148a2de4bdd4a39963a753ffbcab5f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875090302 2024-11-17T20:24:52,871 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c26feba91acd40129c2bd4d891472389, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875090333 2024-11-17T20:24:52,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742087_1263 (size=12001) 2024-11-17T20:24:52,873 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137d3fdc6f7548f7882cafdd0d552385, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731875091498 2024-11-17T20:24:52,873 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/1efcc9176eff447f81ed305a636677f3 2024-11-17T20:24:52,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bf008f5eef62417fa0057828b58e429c is 50, key is test_row_0/B:col10/1731875091574/Put/seqid=0 2024-11-17T20:24:52,882 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:52,882 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/94b4c35f698343fd880f61368de586e2 is 50, key is test_row_0/C:col10/1731875091498/Put/seqid=0 2024-11-17T20:24:52,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742089_1265 (size=12207) 2024-11-17T20:24:52,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742088_1264 (size=12001) 2024-11-17T20:24:52,922 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bf008f5eef62417fa0057828b58e429c 2024-11-17T20:24:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/60963ecef1b7425ca23c7096955ba826 is 50, key is test_row_0/C:col10/1731875091574/Put/seqid=0 2024-11-17T20:24:52,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742090_1266 (size=12001) 2024-11-17T20:24:52,953 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/60963ecef1b7425ca23c7096955ba826 2024-11-17T20:24:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/1efcc9176eff447f81ed305a636677f3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1efcc9176eff447f81ed305a636677f3 2024-11-17T20:24:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1efcc9176eff447f81ed305a636677f3, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T20:24:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bf008f5eef62417fa0057828b58e429c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bf008f5eef62417fa0057828b58e429c 2024-11-17T20:24:52,967 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bf008f5eef62417fa0057828b58e429c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T20:24:52,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/60963ecef1b7425ca23c7096955ba826 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/60963ecef1b7425ca23c7096955ba826 2024-11-17T20:24:52,972 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/60963ecef1b7425ca23c7096955ba826, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T20:24:52,973 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 112ms, sequenceid=114, compaction requested=false 2024-11-17T20:24:52,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:52,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:52,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T20:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T20:24:52,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T20:24:52,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 572 msec 2024-11-17T20:24:52,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 580 msec 2024-11-17T20:24:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T20:24:53,029 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:53,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T20:24:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T20:24:53,031 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:53,032 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:53,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T20:24:53,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T20:24:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T20:24:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T20:24:53,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T20:24:53,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-17T20:24:53,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 159 msec 2024-11-17T20:24:53,273 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bd267cdc7cd84121a8b6c2eac13257ae as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bd267cdc7cd84121a8b6c2eac13257ae 2024-11-17T20:24:53,278 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into bd267cdc7cd84121a8b6c2eac13257ae(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:53,279 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:53,279 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875092807; duration=0sec 2024-11-17T20:24:53,279 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:53,279 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:24:53,314 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/94b4c35f698343fd880f61368de586e2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/94b4c35f698343fd880f61368de586e2 2024-11-17T20:24:53,318 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into 94b4c35f698343fd880f61368de586e2(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:53,318 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:53,318 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=13, startTime=1731875092807; duration=0sec 2024-11-17T20:24:53,319 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:53,319 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:24:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T20:24:53,350 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:53,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T20:24:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T20:24:53,352 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:53,352 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:53,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T20:24:53,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T20:24:53,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T20:24:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T20:24:53,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T20:24:53,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-17T20:24:53,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 160 msec 2024-11-17T20:24:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:53,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:24:53,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/4c81b903d7c84cb786fc93aa1b6c50d2 is 50, key is test_row_1/A:col10/1731875093646/Put/seqid=0 2024-11-17T20:24:53,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742091_1267 (size=9757) 2024-11-17T20:24:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T20:24:53,670 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:53,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-17T20:24:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T20:24:53,671 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:53,672 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:53,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875153681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875153683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875153684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875153729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875153729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,731 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 31e1f74c01abc6e3eb193ea5bbd4a22a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:53,732 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 31e1f74c01abc6e3eb193ea5bbd4a22a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T20:24:53,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875153789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875153789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875153789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:53,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T20:24:53,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:53,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:53,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:53,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T20:24:53,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:53,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:53,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:53,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T20:24:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875153999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875153999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875153999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/4c81b903d7c84cb786fc93aa1b6c50d2 2024-11-17T20:24:54,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/5077b4b3a886439b93bf3cb75e411074 is 50, key is test_row_1/B:col10/1731875093646/Put/seqid=0 2024-11-17T20:24:54,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742092_1268 (size=9757) 2024-11-17T20:24:54,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/5077b4b3a886439b93bf3cb75e411074 2024-11-17T20:24:54,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/1f1b842278cd4ccfb5255ff5566852cb is 50, key is test_row_1/C:col10/1731875093646/Put/seqid=0 2024-11-17T20:24:54,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742093_1269 (size=9757) 2024-11-17T20:24:54,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/1f1b842278cd4ccfb5255ff5566852cb 2024-11-17T20:24:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/4c81b903d7c84cb786fc93aa1b6c50d2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4c81b903d7c84cb786fc93aa1b6c50d2 2024-11-17T20:24:54,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4c81b903d7c84cb786fc93aa1b6c50d2, entries=100, sequenceid=130, filesize=9.5 K 2024-11-17T20:24:54,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T20:24:54,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/5077b4b3a886439b93bf3cb75e411074 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/5077b4b3a886439b93bf3cb75e411074 2024-11-17T20:24:54,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:54,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/5077b4b3a886439b93bf3cb75e411074, entries=100, sequenceid=130, filesize=9.5 K 2024-11-17T20:24:54,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/1f1b842278cd4ccfb5255ff5566852cb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1f1b842278cd4ccfb5255ff5566852cb 2024-11-17T20:24:54,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1f1b842278cd4ccfb5255ff5566852cb, entries=100, sequenceid=130, filesize=9.5 K 2024-11-17T20:24:54,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 500ms, sequenceid=130, compaction requested=true 2024-11-17T20:24:54,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:54,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:54,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:54,147 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:54,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:54,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:54,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:54,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T20:24:54,148 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:54,149 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:54,149 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:24:54,149 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:54,149 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bd267cdc7cd84121a8b6c2eac13257ae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bf008f5eef62417fa0057828b58e429c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/5077b4b3a886439b93bf3cb75e411074] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=33.2 K 2024-11-17T20:24:54,150 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:54,150 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:24:54,150 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:54,150 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/8b8dbe4c9f1b4152b07b73406a5743d8, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1efcc9176eff447f81ed305a636677f3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4c81b903d7c84cb786fc93aa1b6c50d2] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=33.2 K 2024-11-17T20:24:54,150 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8dbe4c9f1b4152b07b73406a5743d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731875091498 2024-11-17T20:24:54,150 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bd267cdc7cd84121a8b6c2eac13257ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731875091498 2024-11-17T20:24:54,151 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1efcc9176eff447f81ed305a636677f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731875091527 2024-11-17T20:24:54,151 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bf008f5eef62417fa0057828b58e429c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731875091527 2024-11-17T20:24:54,151 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c81b903d7c84cb786fc93aa1b6c50d2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875093646 2024-11-17T20:24:54,152 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5077b4b3a886439b93bf3cb75e411074, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875093646 2024-11-17T20:24:54,166 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:54,167 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/32b43282ff264088bca3166afd93d16b is 50, key is test_row_0/A:col10/1731875091574/Put/seqid=0 2024-11-17T20:24:54,170 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:54,171 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/e4304c0c7ed94bc9967c6e255e9313bc is 50, key is test_row_0/B:col10/1731875091574/Put/seqid=0 2024-11-17T20:24:54,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742094_1270 (size=12409) 2024-11-17T20:24:54,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742095_1271 (size=12409) 2024-11-17T20:24:54,259 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/e4304c0c7ed94bc9967c6e255e9313bc as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e4304c0c7ed94bc9967c6e255e9313bc 2024-11-17T20:24:54,268 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into e4304c0c7ed94bc9967c6e255e9313bc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:54,268 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:54,268 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875094147; duration=0sec 2024-11-17T20:24:54,268 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:54,268 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:24:54,268 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:54,270 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:54,270 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:24:54,271 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:54,271 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/94b4c35f698343fd880f61368de586e2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/60963ecef1b7425ca23c7096955ba826, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1f1b842278cd4ccfb5255ff5566852cb] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=33.2 K 2024-11-17T20:24:54,271 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b4c35f698343fd880f61368de586e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731875091498 2024-11-17T20:24:54,271 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 60963ecef1b7425ca23c7096955ba826, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731875091527 2024-11-17T20:24:54,272 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1b842278cd4ccfb5255ff5566852cb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875093646 2024-11-17T20:24:54,286 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:54,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T20:24:54,287 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/3aaeda73db50474fb328ee7b88986eb5 is 50, key is test_row_0/C:col10/1731875091574/Put/seqid=0 2024-11-17T20:24:54,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:54,288 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:24:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a9a039908f694ad58a775a1194efaafd is 50, key is test_row_0/A:col10/1731875093683/Put/seqid=0 2024-11-17T20:24:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T20:24:54,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742096_1272 (size=12409) 2024-11-17T20:24:54,315 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/3aaeda73db50474fb328ee7b88986eb5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3aaeda73db50474fb328ee7b88986eb5 2024-11-17T20:24:54,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:54,322 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into 3aaeda73db50474fb328ee7b88986eb5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:54,322 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:54,322 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=13, startTime=1731875094148; duration=0sec 2024-11-17T20:24:54,322 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:54,322 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:24:54,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742097_1273 (size=12151) 2024-11-17T20:24:54,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875154339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875154340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875154341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875154450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875154450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875154459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,621 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/32b43282ff264088bca3166afd93d16b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32b43282ff264088bca3166afd93d16b 2024-11-17T20:24:54,626 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into 32b43282ff264088bca3166afd93d16b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:54,626 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:54,626 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875094147; duration=0sec 2024-11-17T20:24:54,626 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:54,626 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:24:54,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875154659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875154659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875154670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,738 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a9a039908f694ad58a775a1194efaafd 2024-11-17T20:24:54,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/e5d77edb619a4df28efb93e623bb706b is 50, key is test_row_0/B:col10/1731875093683/Put/seqid=0 2024-11-17T20:24:54,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742098_1274 (size=12151) 2024-11-17T20:24:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T20:24:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875154969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875154969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875154979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,182 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/e5d77edb619a4df28efb93e623bb706b 2024-11-17T20:24:55,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/53b67549518a48b4aa510ea8554b1b67 is 50, key is test_row_0/C:col10/1731875093683/Put/seqid=0 2024-11-17T20:24:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742099_1275 (size=12151) 2024-11-17T20:24:55,244 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/53b67549518a48b4aa510ea8554b1b67 2024-11-17T20:24:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a9a039908f694ad58a775a1194efaafd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a9a039908f694ad58a775a1194efaafd 2024-11-17T20:24:55,256 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a9a039908f694ad58a775a1194efaafd, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T20:24:55,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/e5d77edb619a4df28efb93e623bb706b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e5d77edb619a4df28efb93e623bb706b 2024-11-17T20:24:55,270 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e5d77edb619a4df28efb93e623bb706b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T20:24:55,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/53b67549518a48b4aa510ea8554b1b67 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/53b67549518a48b4aa510ea8554b1b67 2024-11-17T20:24:55,279 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/53b67549518a48b4aa510ea8554b1b67, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T20:24:55,281 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 993ms, sequenceid=156, compaction requested=false 2024-11-17T20:24:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-17T20:24:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-17T20:24:55,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T20:24:55,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-11-17T20:24:55,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6160 sec 2024-11-17T20:24:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:55,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:24:55,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/00ef5e2d630049e3acbe968085ad4357 is 50, key is test_row_0/A:col10/1731875095483/Put/seqid=0 2024-11-17T20:24:55,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742100_1276 (size=12151) 2024-11-17T20:24:55,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/00ef5e2d630049e3acbe968085ad4357 2024-11-17T20:24:55,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/89f749ffc6d6408f9dc27996bf05b0b7 is 50, key is test_row_0/B:col10/1731875095483/Put/seqid=0 2024-11-17T20:24:55,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742101_1277 (size=12151) 2024-11-17T20:24:55,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/89f749ffc6d6408f9dc27996bf05b0b7 2024-11-17T20:24:55,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875155535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/12b3e2935ff64adebc817c5c1d7dc3d6 is 50, key is test_row_0/C:col10/1731875095483/Put/seqid=0 2024-11-17T20:24:55,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875155540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875155540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742102_1278 (size=12151) 2024-11-17T20:24:55,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/12b3e2935ff64adebc817c5c1d7dc3d6 2024-11-17T20:24:55,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/00ef5e2d630049e3acbe968085ad4357 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/00ef5e2d630049e3acbe968085ad4357 2024-11-17T20:24:55,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/00ef5e2d630049e3acbe968085ad4357, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T20:24:55,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/89f749ffc6d6408f9dc27996bf05b0b7 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/89f749ffc6d6408f9dc27996bf05b0b7 2024-11-17T20:24:55,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/89f749ffc6d6408f9dc27996bf05b0b7, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T20:24:55,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/12b3e2935ff64adebc817c5c1d7dc3d6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/12b3e2935ff64adebc817c5c1d7dc3d6 2024-11-17T20:24:55,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/12b3e2935ff64adebc817c5c1d7dc3d6, entries=150, sequenceid=172, filesize=11.9 K 2024-11-17T20:24:55,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 88ms, sequenceid=172, compaction requested=true 2024-11-17T20:24:55,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:55,575 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:55,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:55,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:55,576 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:55,580 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:55,580 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:55,580 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:24:55,580 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:24:55,580 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:55,580 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:55,580 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32b43282ff264088bca3166afd93d16b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a9a039908f694ad58a775a1194efaafd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/00ef5e2d630049e3acbe968085ad4357] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.9 K 2024-11-17T20:24:55,580 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e4304c0c7ed94bc9967c6e255e9313bc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e5d77edb619a4df28efb93e623bb706b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/89f749ffc6d6408f9dc27996bf05b0b7] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.9 K 2024-11-17T20:24:55,581 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32b43282ff264088bca3166afd93d16b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875091574 2024-11-17T20:24:55,581 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e4304c0c7ed94bc9967c6e255e9313bc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875091574 2024-11-17T20:24:55,581 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a039908f694ad58a775a1194efaafd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731875093679 2024-11-17T20:24:55,582 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d77edb619a4df28efb93e623bb706b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731875093679 2024-11-17T20:24:55,582 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00ef5e2d630049e3acbe968085ad4357, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875094326 2024-11-17T20:24:55,582 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f749ffc6d6408f9dc27996bf05b0b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875094326 2024-11-17T20:24:55,592 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:55,593 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/79be3520f3574f6a8ae380d32f73d04d is 50, key is test_row_0/B:col10/1731875095483/Put/seqid=0 2024-11-17T20:24:55,593 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:55,594 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/6e6ca638d66f441798e170fc8c022edb is 50, key is test_row_0/A:col10/1731875095483/Put/seqid=0 2024-11-17T20:24:55,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742103_1279 (size=12561) 2024-11-17T20:24:55,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742104_1280 (size=12561) 2024-11-17T20:24:55,646 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/79be3520f3574f6a8ae380d32f73d04d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/79be3520f3574f6a8ae380d32f73d04d 2024-11-17T20:24:55,659 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into 79be3520f3574f6a8ae380d32f73d04d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:55,660 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:55,660 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875095576; duration=0sec 2024-11-17T20:24:55,660 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:55,660 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:24:55,660 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:55,660 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:55,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:24:55,662 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:55,662 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:24:55,662 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:55,662 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3aaeda73db50474fb328ee7b88986eb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/53b67549518a48b4aa510ea8554b1b67, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/12b3e2935ff64adebc817c5c1d7dc3d6] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=35.9 K 2024-11-17T20:24:55,663 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aaeda73db50474fb328ee7b88986eb5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731875091574 2024-11-17T20:24:55,663 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b67549518a48b4aa510ea8554b1b67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731875093679 2024-11-17T20:24:55,664 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b3e2935ff64adebc817c5c1d7dc3d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875094326 2024-11-17T20:24:55,682 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:55,683 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c7bf463c1cc34914b4340dd27c376e82 is 50, key is test_row_0/C:col10/1731875095483/Put/seqid=0 2024-11-17T20:24:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875155683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875155688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875155688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/c2ae2b4416e94a139be32f473a9e4b24 is 50, key is test_row_0/A:col10/1731875095532/Put/seqid=0 2024-11-17T20:24:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742105_1281 (size=12561) 2024-11-17T20:24:55,716 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/c7bf463c1cc34914b4340dd27c376e82 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c7bf463c1cc34914b4340dd27c376e82 2024-11-17T20:24:55,724 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into c7bf463c1cc34914b4340dd27c376e82(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:55,724 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:55,725 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=13, startTime=1731875095576; duration=0sec 2024-11-17T20:24:55,725 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:55,725 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:24:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875155739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,743 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 31e1f74c01abc6e3eb193ea5bbd4a22a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875155749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742106_1282 (size=12151) 2024-11-17T20:24:55,754 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 31e1f74c01abc6e3eb193ea5bbd4a22a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/c2ae2b4416e94a139be32f473a9e4b24 2024-11-17T20:24:55,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/82265ffb24f44797b8db06aa3c990629 is 50, key is test_row_0/B:col10/1731875095532/Put/seqid=0 2024-11-17T20:24:55,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742107_1283 (size=12151) 2024-11-17T20:24:55,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/82265ffb24f44797b8db06aa3c990629 2024-11-17T20:24:55,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/d7d3f7249c824e7ea1c6123d08a306f7 is 50, key is test_row_0/C:col10/1731875095532/Put/seqid=0 2024-11-17T20:24:55,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875155799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875155799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875155799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T20:24:55,821 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:55,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-17T20:24:55,824 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:55,826 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:55,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T20:24:55,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742108_1284 (size=12151) 2024-11-17T20:24:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T20:24:55,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T20:24:55,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:55,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:56,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875156010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875156010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875156010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,024 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/6e6ca638d66f441798e170fc8c022edb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/6e6ca638d66f441798e170fc8c022edb 2024-11-17T20:24:56,031 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into 6e6ca638d66f441798e170fc8c022edb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:56,031 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:56,031 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875095575; duration=0sec 2024-11-17T20:24:56,031 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:56,031 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:24:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T20:24:56,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T20:24:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:56,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:56,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/d7d3f7249c824e7ea1c6123d08a306f7 2024-11-17T20:24:56,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/c2ae2b4416e94a139be32f473a9e4b24 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2ae2b4416e94a139be32f473a9e4b24 2024-11-17T20:24:56,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2ae2b4416e94a139be32f473a9e4b24, entries=150, sequenceid=198, filesize=11.9 K 2024-11-17T20:24:56,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/82265ffb24f44797b8db06aa3c990629 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/82265ffb24f44797b8db06aa3c990629 2024-11-17T20:24:56,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/82265ffb24f44797b8db06aa3c990629, entries=150, sequenceid=198, filesize=11.9 K 2024-11-17T20:24:56,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/d7d3f7249c824e7ea1c6123d08a306f7 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d7d3f7249c824e7ea1c6123d08a306f7 2024-11-17T20:24:56,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d7d3f7249c824e7ea1c6123d08a306f7, entries=150, sequenceid=198, filesize=11.9 K 2024-11-17T20:24:56,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 606ms, sequenceid=198, compaction requested=false 2024-11-17T20:24:56,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:56,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T20:24:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:56,297 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:24:56,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/40988e201481440c83cecb44015da4cc is 50, key is test_row_0/A:col10/1731875095671/Put/seqid=0 2024-11-17T20:24:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:56,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:56,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742109_1285 (size=12151) 2024-11-17T20:24:56,358 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/40988e201481440c83cecb44015da4cc 2024-11-17T20:24:56,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875156365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/557139d0296547daaeb885470a2a7191 is 50, key is test_row_0/B:col10/1731875095671/Put/seqid=0 2024-11-17T20:24:56,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875156369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875156369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742110_1286 (size=12151) 2024-11-17T20:24:56,406 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/557139d0296547daaeb885470a2a7191 2024-11-17T20:24:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/44e18ce6ccb8430c9977587c23588016 is 50, key is test_row_0/C:col10/1731875095671/Put/seqid=0 2024-11-17T20:24:56,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742111_1287 (size=12151) 2024-11-17T20:24:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T20:24:56,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875156482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875156484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875156484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875156689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875156689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875156689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:56,848 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/44e18ce6ccb8430c9977587c23588016 2024-11-17T20:24:56,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/40988e201481440c83cecb44015da4cc as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/40988e201481440c83cecb44015da4cc 2024-11-17T20:24:56,859 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/40988e201481440c83cecb44015da4cc, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T20:24:56,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/557139d0296547daaeb885470a2a7191 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/557139d0296547daaeb885470a2a7191 2024-11-17T20:24:56,866 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/557139d0296547daaeb885470a2a7191, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T20:24:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/44e18ce6ccb8430c9977587c23588016 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/44e18ce6ccb8430c9977587c23588016 2024-11-17T20:24:56,871 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/44e18ce6ccb8430c9977587c23588016, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T20:24:56,871 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 574ms, sequenceid=212, compaction requested=true 2024-11-17T20:24:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:56,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-17T20:24:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-17T20:24:56,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T20:24:56,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-11-17T20:24:56,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0530 sec 2024-11-17T20:24:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T20:24:56,959 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:56,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-17T20:24:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T20:24:56,961 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:56,962 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:56,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:57,002 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:57,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:24:57,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/d5c49bbcee1d4f809b4fad9dadfdb825 is 50, key is test_row_0/A:col10/1731875096367/Put/seqid=0 2024-11-17T20:24:57,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742112_1288 (size=12151) 2024-11-17T20:24:57,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875157013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875157014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/d5c49bbcee1d4f809b4fad9dadfdb825 2024-11-17T20:24:57,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875157016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5 is 50, key is test_row_0/B:col10/1731875096367/Put/seqid=0 2024-11-17T20:24:57,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742113_1289 (size=12151) 2024-11-17T20:24:57,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5 2024-11-17T20:24:57,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/86c837ae53cf44128db7d0584c00ddec is 50, key is test_row_0/C:col10/1731875096367/Put/seqid=0 2024-11-17T20:24:57,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742114_1290 (size=12151) 2024-11-17T20:24:57,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/86c837ae53cf44128db7d0584c00ddec 2024-11-17T20:24:57,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/d5c49bbcee1d4f809b4fad9dadfdb825 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d5c49bbcee1d4f809b4fad9dadfdb825 2024-11-17T20:24:57,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d5c49bbcee1d4f809b4fad9dadfdb825, entries=150, sequenceid=237, filesize=11.9 K 2024-11-17T20:24:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T20:24:57,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5 2024-11-17T20:24:57,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5, entries=150, sequenceid=237, filesize=11.9 K 2024-11-17T20:24:57,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/86c837ae53cf44128db7d0584c00ddec as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/86c837ae53cf44128db7d0584c00ddec 2024-11-17T20:24:57,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/86c837ae53cf44128db7d0584c00ddec, entries=150, sequenceid=237, filesize=11.9 K 2024-11-17T20:24:57,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 81ms, sequenceid=237, compaction requested=true 2024-11-17T20:24:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:57,083 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:57,083 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:24:57,086 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:57,086 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:24:57,086 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,086 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/79be3520f3574f6a8ae380d32f73d04d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/82265ffb24f44797b8db06aa3c990629, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/557139d0296547daaeb885470a2a7191, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=47.9 K 2024-11-17T20:24:57,086 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:24:57,086 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:24:57,086 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,086 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/6e6ca638d66f441798e170fc8c022edb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2ae2b4416e94a139be32f473a9e4b24, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/40988e201481440c83cecb44015da4cc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d5c49bbcee1d4f809b4fad9dadfdb825] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=47.9 K 2024-11-17T20:24:57,087 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 79be3520f3574f6a8ae380d32f73d04d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875094326 2024-11-17T20:24:57,088 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6ca638d66f441798e170fc8c022edb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875094326 2024-11-17T20:24:57,088 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 82265ffb24f44797b8db06aa3c990629, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731875095532 2024-11-17T20:24:57,089 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ae2b4416e94a139be32f473a9e4b24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731875095532 2024-11-17T20:24:57,089 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 557139d0296547daaeb885470a2a7191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731875095671 2024-11-17T20:24:57,090 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40988e201481440c83cecb44015da4cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731875095671 2024-11-17T20:24:57,090 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ce9f9ee77d4047b6a61a3ab7a7b6e2d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731875096362 2024-11-17T20:24:57,093 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5c49bbcee1d4f809b4fad9dadfdb825, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731875096362 2024-11-17T20:24:57,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T20:24:57,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,114 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:24:57,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:57,131 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:57,131 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/6b8d4877cba74894aa6fbef2078190b9 is 50, key is test_row_0/B:col10/1731875096367/Put/seqid=0 2024-11-17T20:24:57,134 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:57,135 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/e44fabbb23c949f6bbe05c9718eb9fb3 is 50, key is test_row_0/A:col10/1731875096367/Put/seqid=0 2024-11-17T20:24:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/49f89769da9942d1adceef8c6846cc88 is 50, key is test_row_0/A:col10/1731875097011/Put/seqid=0 2024-11-17T20:24:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742115_1291 (size=12697) 2024-11-17T20:24:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742116_1292 (size=12697) 2024-11-17T20:24:57,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875157179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875157179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875157184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742117_1293 (size=12151) 2024-11-17T20:24:57,204 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/49f89769da9942d1adceef8c6846cc88 2024-11-17T20:24:57,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/f5e7be1ec025465ea0a9db0430ae3652 is 50, key is test_row_0/B:col10/1731875097011/Put/seqid=0 2024-11-17T20:24:57,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742118_1294 (size=12151) 2024-11-17T20:24:57,237 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/f5e7be1ec025465ea0a9db0430ae3652 2024-11-17T20:24:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/762142de797648c08485432a08b92d94 is 50, key is test_row_0/C:col10/1731875097011/Put/seqid=0 2024-11-17T20:24:57,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742119_1295 (size=12151) 2024-11-17T20:24:57,272 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/762142de797648c08485432a08b92d94 2024-11-17T20:24:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T20:24:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/49f89769da9942d1adceef8c6846cc88 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/49f89769da9942d1adceef8c6846cc88 2024-11-17T20:24:57,290 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/49f89769da9942d1adceef8c6846cc88, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T20:24:57,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/f5e7be1ec025465ea0a9db0430ae3652 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f5e7be1ec025465ea0a9db0430ae3652 2024-11-17T20:24:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875157289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875157289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,300 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f5e7be1ec025465ea0a9db0430ae3652, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T20:24:57,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/762142de797648c08485432a08b92d94 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/762142de797648c08485432a08b92d94 2024-11-17T20:24:57,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875157300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,310 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/762142de797648c08485432a08b92d94, entries=150, sequenceid=248, filesize=11.9 K 2024-11-17T20:24:57,311 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 197ms, sequenceid=248, compaction requested=true 2024-11-17T20:24:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-17T20:24:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-17T20:24:57,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-17T20:24:57,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 351 msec 2024-11-17T20:24:57,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 356 msec 2024-11-17T20:24:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:57,502 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:57,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T20:24:57,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/b7404944950c4cce80fcf7c45b3d0a42 is 50, key is test_row_0/A:col10/1731875097501/Put/seqid=0 2024-11-17T20:24:57,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875157516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875157516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875157517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742120_1296 (size=14741) 2024-11-17T20:24:57,585 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/e44fabbb23c949f6bbe05c9718eb9fb3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e44fabbb23c949f6bbe05c9718eb9fb3 2024-11-17T20:24:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T20:24:57,590 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:24:57,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:24:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-17T20:24:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T20:24:57,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:24:57,593 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:24:57,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:24:57,595 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into e44fabbb23c949f6bbe05c9718eb9fb3(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:57,595 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:57,595 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=12, startTime=1731875097083; duration=0sec 2024-11-17T20:24:57,595 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:57,595 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:24:57,596 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T20:24:57,601 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/6b8d4877cba74894aa6fbef2078190b9 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/6b8d4877cba74894aa6fbef2078190b9 2024-11-17T20:24:57,601 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T20:24:57,601 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:24:57,601 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,601 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c7bf463c1cc34914b4340dd27c376e82, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d7d3f7249c824e7ea1c6123d08a306f7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/44e18ce6ccb8430c9977587c23588016, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/86c837ae53cf44128db7d0584c00ddec, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/762142de797648c08485432a08b92d94] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=59.7 K 2024-11-17T20:24:57,602 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7bf463c1cc34914b4340dd27c376e82, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731875094326 2024-11-17T20:24:57,602 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d3f7249c824e7ea1c6123d08a306f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731875095532 2024-11-17T20:24:57,602 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e18ce6ccb8430c9977587c23588016, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731875095671 2024-11-17T20:24:57,603 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86c837ae53cf44128db7d0584c00ddec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731875096362 2024-11-17T20:24:57,603 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 762142de797648c08485432a08b92d94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731875097011 2024-11-17T20:24:57,606 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into 6b8d4877cba74894aa6fbef2078190b9(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:57,606 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:57,606 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=12, startTime=1731875097083; duration=0sec 2024-11-17T20:24:57,606 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:57,606 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:24:57,629 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#248 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:57,630 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/082f34f23afb4187ba7206aa5d0f68c6 is 50, key is test_row_0/C:col10/1731875097011/Put/seqid=0 2024-11-17T20:24:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875157629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875157629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875157630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742121_1297 (size=12731) 2024-11-17T20:24:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T20:24:57,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875157840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875157840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875157840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:57,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:57,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:57,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:57,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:57,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T20:24:57,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/b7404944950c4cce80fcf7c45b3d0a42 2024-11-17T20:24:57,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/359fabe140394a14a935598ee850009a is 50, key is test_row_0/B:col10/1731875097501/Put/seqid=0 2024-11-17T20:24:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742122_1298 (size=12301) 2024-11-17T20:24:58,049 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/082f34f23afb4187ba7206aa5d0f68c6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/082f34f23afb4187ba7206aa5d0f68c6 2024-11-17T20:24:58,053 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into 082f34f23afb4187ba7206aa5d0f68c6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:58,053 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:58,053 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=11, startTime=1731875097083; duration=0sec 2024-11-17T20:24:58,053 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:58,054 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:24:58,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875158149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875158149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875158149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:58,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:58,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:58,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T20:24:58,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/359fabe140394a14a935598ee850009a 2024-11-17T20:24:58,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/3e01c49e1c0f4c408e15c34eaf2ebabd is 50, key is test_row_0/C:col10/1731875097501/Put/seqid=0 2024-11-17T20:24:58,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742123_1299 (size=12301) 2024-11-17T20:24:58,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:58,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:58,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875158659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:58,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875158659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875158660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:58,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:24:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T20:24:58,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/3e01c49e1c0f4c408e15c34eaf2ebabd 2024-11-17T20:24:58,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/b7404944950c4cce80fcf7c45b3d0a42 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/b7404944950c4cce80fcf7c45b3d0a42 2024-11-17T20:24:58,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/b7404944950c4cce80fcf7c45b3d0a42, entries=200, sequenceid=274, filesize=14.4 K 2024-11-17T20:24:58,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/359fabe140394a14a935598ee850009a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/359fabe140394a14a935598ee850009a 2024-11-17T20:24:58,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/359fabe140394a14a935598ee850009a, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T20:24:58,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/3e01c49e1c0f4c408e15c34eaf2ebabd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3e01c49e1c0f4c408e15c34eaf2ebabd 2024-11-17T20:24:58,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3e01c49e1c0f4c408e15c34eaf2ebabd, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T20:24:58,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1293ms, sequenceid=274, compaction requested=true 2024-11-17T20:24:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:24:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:58,795 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:24:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:58,795 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:24:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:24:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:58,796 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:58,796 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:24:58,796 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:24:58,796 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:24:58,796 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,796 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,796 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e44fabbb23c949f6bbe05c9718eb9fb3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/49f89769da9942d1adceef8c6846cc88, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/b7404944950c4cce80fcf7c45b3d0a42] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=38.7 K 2024-11-17T20:24:58,796 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/6b8d4877cba74894aa6fbef2078190b9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f5e7be1ec025465ea0a9db0430ae3652, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/359fabe140394a14a935598ee850009a] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=36.3 K 2024-11-17T20:24:58,796 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e44fabbb23c949f6bbe05c9718eb9fb3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731875096362 2024-11-17T20:24:58,796 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8d4877cba74894aa6fbef2078190b9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731875096362 2024-11-17T20:24:58,797 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f89769da9942d1adceef8c6846cc88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731875097011 2024-11-17T20:24:58,797 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e7be1ec025465ea0a9db0430ae3652, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731875097011 2024-11-17T20:24:58,797 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7404944950c4cce80fcf7c45b3d0a42, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731875097177 2024-11-17T20:24:58,797 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 359fabe140394a14a935598ee850009a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731875097177 2024-11-17T20:24:58,814 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:58,815 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/20164bb959d74454a8f3be4e5656ac47 is 50, key is test_row_0/B:col10/1731875097501/Put/seqid=0 2024-11-17T20:24:58,818 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:24:58,819 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a64e0ecc4449492180ed8c89e3b3a74f is 50, key is test_row_0/A:col10/1731875097501/Put/seqid=0 2024-11-17T20:24:58,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T20:24:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:58,821 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T20:24:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/4ea6d30865454882824c55393cd95556 is 50, key is test_row_0/A:col10/1731875097502/Put/seqid=0 2024-11-17T20:24:58,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742125_1301 (size=12949) 2024-11-17T20:24:58,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742124_1300 (size=12949) 2024-11-17T20:24:58,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742126_1302 (size=12301) 2024-11-17T20:24:59,240 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a64e0ecc4449492180ed8c89e3b3a74f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a64e0ecc4449492180ed8c89e3b3a74f 2024-11-17T20:24:59,245 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into a64e0ecc4449492180ed8c89e3b3a74f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:59,245 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:59,245 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875098795; duration=0sec 2024-11-17T20:24:59,245 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:24:59,245 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:24:59,246 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T20:24:59,246 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T20:24:59,246 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T20:24:59,246 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. because compaction request was cancelled 2024-11-17T20:24:59,246 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:24:59,252 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/20164bb959d74454a8f3be4e5656ac47 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/20164bb959d74454a8f3be4e5656ac47 2024-11-17T20:24:59,256 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/4ea6d30865454882824c55393cd95556 2024-11-17T20:24:59,261 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into 20164bb959d74454a8f3be4e5656ac47(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:24:59,261 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:59,261 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875098795; duration=0sec 2024-11-17T20:24:59,262 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:24:59,262 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:24:59,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/078880e5da42481d94bf9a84cfe427ba is 50, key is test_row_0/B:col10/1731875097502/Put/seqid=0 2024-11-17T20:24:59,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742127_1303 (size=12301) 2024-11-17T20:24:59,272 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/078880e5da42481d94bf9a84cfe427ba 2024-11-17T20:24:59,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/5a71df34e8bb44b09b1e48373386f4c6 is 50, key is test_row_0/C:col10/1731875097502/Put/seqid=0 2024-11-17T20:24:59,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742128_1304 (size=12301) 2024-11-17T20:24:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:24:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:59,688 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/5a71df34e8bb44b09b1e48373386f4c6 2024-11-17T20:24:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/4ea6d30865454882824c55393cd95556 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4ea6d30865454882824c55393cd95556 2024-11-17T20:24:59,702 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4ea6d30865454882824c55393cd95556, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T20:24:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/078880e5da42481d94bf9a84cfe427ba as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/078880e5da42481d94bf9a84cfe427ba 2024-11-17T20:24:59,708 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/078880e5da42481d94bf9a84cfe427ba, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T20:24:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/5a71df34e8bb44b09b1e48373386f4c6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/5a71df34e8bb44b09b1e48373386f4c6 2024-11-17T20:24:59,728 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/5a71df34e8bb44b09b1e48373386f4c6, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T20:24:59,734 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=140.89 KB/144270 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 912ms, sequenceid=287, compaction requested=true 2024-11-17T20:24:59,734 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:24:59,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:24:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:24:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:24:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-17T20:24:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-17T20:24:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:24:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T20:24:59,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T20:24:59,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1430 sec 2024-11-17T20:24:59,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.1490 sec 2024-11-17T20:24:59,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/38bd24066603479199ab39ae89dbf651 is 50, key is test_row_0/A:col10/1731875099733/Put/seqid=0 2024-11-17T20:24:59,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875159753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875159753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875159755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875159754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38988 deadline: 1731875159770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,773 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 31e1f74c01abc6e3eb193ea5bbd4a22a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:24:59,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742129_1305 (size=14741) 2024-11-17T20:24:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875159869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875159869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875159869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875159869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:24:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:24:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875160079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875160079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875160079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875160080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/38bd24066603479199ab39ae89dbf651 2024-11-17T20:25:00,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/521850e5a0944979a0d2971da050b0f1 is 50, key is test_row_0/B:col10/1731875099733/Put/seqid=0 2024-11-17T20:25:00,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742130_1306 (size=12301) 2024-11-17T20:25:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875160389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875160389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875160389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875160389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/521850e5a0944979a0d2971da050b0f1 2024-11-17T20:25:00,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/e0e62ec1180346bfa7333970e8d15a36 is 50, key is test_row_0/C:col10/1731875099733/Put/seqid=0 2024-11-17T20:25:00,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742131_1307 (size=12301) 2024-11-17T20:25:00,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/e0e62ec1180346bfa7333970e8d15a36 2024-11-17T20:25:00,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/38bd24066603479199ab39ae89dbf651 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/38bd24066603479199ab39ae89dbf651 2024-11-17T20:25:00,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/38bd24066603479199ab39ae89dbf651, entries=200, sequenceid=314, filesize=14.4 K 2024-11-17T20:25:00,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/521850e5a0944979a0d2971da050b0f1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/521850e5a0944979a0d2971da050b0f1 2024-11-17T20:25:00,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/521850e5a0944979a0d2971da050b0f1, entries=150, sequenceid=314, filesize=12.0 K 2024-11-17T20:25:00,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/e0e62ec1180346bfa7333970e8d15a36 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e0e62ec1180346bfa7333970e8d15a36 2024-11-17T20:25:00,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e0e62ec1180346bfa7333970e8d15a36, entries=150, sequenceid=314, filesize=12.0 K 2024-11-17T20:25:00,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 937ms, sequenceid=314, compaction requested=true 2024-11-17T20:25:00,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:00,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:00,671 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:00,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:00,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:00,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:00,671 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:00,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:00,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:00,672 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:00,672 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:25:00,672 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:00,673 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/20164bb959d74454a8f3be4e5656ac47, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/078880e5da42481d94bf9a84cfe427ba, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/521850e5a0944979a0d2971da050b0f1] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=36.7 K 2024-11-17T20:25:00,673 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:00,673 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:25:00,673 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:00,673 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a64e0ecc4449492180ed8c89e3b3a74f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4ea6d30865454882824c55393cd95556, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/38bd24066603479199ab39ae89dbf651] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=39.1 K 2024-11-17T20:25:00,673 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a64e0ecc4449492180ed8c89e3b3a74f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731875097177 2024-11-17T20:25:00,673 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 20164bb959d74454a8f3be4e5656ac47, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731875097177 2024-11-17T20:25:00,674 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea6d30865454882824c55393cd95556, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731875097502 2024-11-17T20:25:00,674 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 078880e5da42481d94bf9a84cfe427ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731875097502 2024-11-17T20:25:00,674 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38bd24066603479199ab39ae89dbf651, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875099703 2024-11-17T20:25:00,674 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 521850e5a0944979a0d2971da050b0f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875099722 2024-11-17T20:25:00,681 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:00,681 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/f24e5b9c67d34b4ebc4374ea77036559 is 50, key is test_row_0/B:col10/1731875099733/Put/seqid=0 2024-11-17T20:25:00,690 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:00,691 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/c2f394e5cb85459c890d7491c8098011 is 50, key is test_row_0/A:col10/1731875099733/Put/seqid=0 2024-11-17T20:25:00,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742132_1308 (size=13051) 2024-11-17T20:25:00,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742133_1309 (size=13051) 2024-11-17T20:25:00,756 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/c2f394e5cb85459c890d7491c8098011 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2f394e5cb85459c890d7491c8098011 2024-11-17T20:25:00,763 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into c2f394e5cb85459c890d7491c8098011(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:00,763 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875100671; duration=0sec 2024-11-17T20:25:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:25:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:00,765 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:00,765 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:25:00,765 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:00,766 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/082f34f23afb4187ba7206aa5d0f68c6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3e01c49e1c0f4c408e15c34eaf2ebabd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/5a71df34e8bb44b09b1e48373386f4c6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e0e62ec1180346bfa7333970e8d15a36] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=48.5 K 2024-11-17T20:25:00,766 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 082f34f23afb4187ba7206aa5d0f68c6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731875097011 2024-11-17T20:25:00,766 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e01c49e1c0f4c408e15c34eaf2ebabd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731875097177 2024-11-17T20:25:00,767 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a71df34e8bb44b09b1e48373386f4c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731875097502 2024-11-17T20:25:00,767 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e62ec1180346bfa7333970e8d15a36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875099722 2024-11-17T20:25:00,778 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:00,779 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/3d9c2bb4f6904095b525bae6ae1c96fe is 50, key is test_row_0/C:col10/1731875099733/Put/seqid=0 2024-11-17T20:25:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742134_1310 (size=13017) 2024-11-17T20:25:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:00,900 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:00,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T20:25:00,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/1bd04fb5d98c445e9b5172c3c98d247e is 50, key is test_row_0/A:col10/1731875099753/Put/seqid=0 2024-11-17T20:25:00,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742135_1311 (size=12301) 2024-11-17T20:25:00,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875160922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875160923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875160926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875160926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875161029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875161029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875161040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875161040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,122 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/f24e5b9c67d34b4ebc4374ea77036559 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f24e5b9c67d34b4ebc4374ea77036559 2024-11-17T20:25:01,140 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into f24e5b9c67d34b4ebc4374ea77036559(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:01,140 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:01,140 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875100671; duration=0sec 2024-11-17T20:25:01,140 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:01,140 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:25:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875161240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875161240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,243 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/3d9c2bb4f6904095b525bae6ae1c96fe as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3d9c2bb4f6904095b525bae6ae1c96fe 2024-11-17T20:25:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875161249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,252 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into 3d9c2bb4f6904095b525bae6ae1c96fe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:01,252 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:01,253 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=12, startTime=1731875100671; duration=0sec 2024-11-17T20:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,253 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:01,253 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875161249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/1bd04fb5d98c445e9b5172c3c98d247e 2024-11-17T20:25:01,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bcad5ea153474f349baa27fc373d0752 is 50, key is test_row_0/B:col10/1731875099753/Put/seqid=0 2024-11-17T20:25:01,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742136_1312 (size=12301) 2024-11-17T20:25:01,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875161549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875161549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875161559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875161560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T20:25:01,750 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:01,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-17T20:25:01,752 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T20:25:01,753 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:01,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:01,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bcad5ea153474f349baa27fc373d0752 2024-11-17T20:25:01,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/bc58536da68c4121a2fce60edc7d4beb is 50, key is test_row_0/C:col10/1731875099753/Put/seqid=0 2024-11-17T20:25:01,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742137_1313 (size=12301) 2024-11-17T20:25:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T20:25:01,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T20:25:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:01,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:01,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:01,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:01,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875162059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875162059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:02,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T20:25:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:02,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T20:25:02,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875162069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875162069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:02,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T20:25:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:02,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:02,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/bc58536da68c4121a2fce60edc7d4beb 2024-11-17T20:25:02,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/1bd04fb5d98c445e9b5172c3c98d247e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1bd04fb5d98c445e9b5172c3c98d247e 2024-11-17T20:25:02,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1bd04fb5d98c445e9b5172c3c98d247e, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:02,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/bcad5ea153474f349baa27fc373d0752 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bcad5ea153474f349baa27fc373d0752 2024-11-17T20:25:02,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bcad5ea153474f349baa27fc373d0752, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/bc58536da68c4121a2fce60edc7d4beb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/bc58536da68c4121a2fce60edc7d4beb 2024-11-17T20:25:02,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/bc58536da68c4121a2fce60edc7d4beb, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:02,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1356ms, sequenceid=328, compaction requested=false 2024-11-17T20:25:02,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:02,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-17T20:25:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:02,370 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T20:25:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/050f57cf985a48178360a8b6c510de08 is 50, key is test_row_0/A:col10/1731875100903/Put/seqid=0 2024-11-17T20:25:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T20:25:02,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742138_1314 (size=12301) 2024-11-17T20:25:02,794 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/050f57cf985a48178360a8b6c510de08 2024-11-17T20:25:02,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/534594f837c743fcbf086c2c16e1aae6 is 50, key is test_row_0/B:col10/1731875100903/Put/seqid=0 2024-11-17T20:25:02,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742139_1315 (size=12301) 2024-11-17T20:25:02,819 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/534594f837c743fcbf086c2c16e1aae6 2024-11-17T20:25:02,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/85f64025ca0240e6964332776f68efd5 is 50, key is test_row_0/C:col10/1731875100903/Put/seqid=0 2024-11-17T20:25:02,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742140_1316 (size=12301) 2024-11-17T20:25:02,864 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/85f64025ca0240e6964332776f68efd5 2024-11-17T20:25:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/050f57cf985a48178360a8b6c510de08 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/050f57cf985a48178360a8b6c510de08 2024-11-17T20:25:02,884 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/050f57cf985a48178360a8b6c510de08, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T20:25:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/534594f837c743fcbf086c2c16e1aae6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/534594f837c743fcbf086c2c16e1aae6 2024-11-17T20:25:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T20:25:02,893 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/534594f837c743fcbf086c2c16e1aae6, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T20:25:02,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/85f64025ca0240e6964332776f68efd5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85f64025ca0240e6964332776f68efd5 2024-11-17T20:25:02,903 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85f64025ca0240e6964332776f68efd5, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T20:25:02,904 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 534ms, sequenceid=353, compaction requested=true 2024-11-17T20:25:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-17T20:25:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-17T20:25:02,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T20:25:02,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1560 sec 2024-11-17T20:25:02,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.1630 sec 2024-11-17T20:25:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:03,085 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:03,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:25:03,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/160fa7a0e03b4b03a4104228d77fd66a is 50, key is test_row_0/A:col10/1731875103075/Put/seqid=0 2024-11-17T20:25:03,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742141_1317 (size=14741) 2024-11-17T20:25:03,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875163118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875163119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875163120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875163120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875163230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875163230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875163230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875163230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875163440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875163440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875163440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875163440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/160fa7a0e03b4b03a4104228d77fd66a 2024-11-17T20:25:03,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/16528f2f6e5b4622878612520a442aff is 50, key is test_row_0/B:col10/1731875103075/Put/seqid=0 2024-11-17T20:25:03,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742142_1318 (size=12301) 2024-11-17T20:25:03,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875163749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875163749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875163749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875163749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T20:25:03,900 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:03,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-17T20:25:03,901 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T20:25:03,902 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:03,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:03,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/16528f2f6e5b4622878612520a442aff 2024-11-17T20:25:03,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/e93fedcf33984fd888de1bf14c3332d4 is 50, key is test_row_0/C:col10/1731875103075/Put/seqid=0 2024-11-17T20:25:03,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742143_1319 (size=12301) 2024-11-17T20:25:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T20:25:04,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T20:25:04,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:04,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:04,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:04,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T20:25:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:04,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T20:25:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875164259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875164259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875164260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:04,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875164260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:04,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/e93fedcf33984fd888de1bf14c3332d4 2024-11-17T20:25:04,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/160fa7a0e03b4b03a4104228d77fd66a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/160fa7a0e03b4b03a4104228d77fd66a 2024-11-17T20:25:04,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/160fa7a0e03b4b03a4104228d77fd66a, entries=200, sequenceid=364, filesize=14.4 K 2024-11-17T20:25:04,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/16528f2f6e5b4622878612520a442aff as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/16528f2f6e5b4622878612520a442aff 2024-11-17T20:25:04,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/16528f2f6e5b4622878612520a442aff, entries=150, sequenceid=364, filesize=12.0 K 2024-11-17T20:25:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/e93fedcf33984fd888de1bf14c3332d4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e93fedcf33984fd888de1bf14c3332d4 2024-11-17T20:25:04,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e93fedcf33984fd888de1bf14c3332d4, entries=150, sequenceid=364, filesize=12.0 K 2024-11-17T20:25:04,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1269ms, sequenceid=364, compaction requested=true 2024-11-17T20:25:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:04,354 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:04,355 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:04,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:04,355 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:04,355 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:04,355 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:25:04,355 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:25:04,356 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,356 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,356 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f24e5b9c67d34b4ebc4374ea77036559, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bcad5ea153474f349baa27fc373d0752, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/534594f837c743fcbf086c2c16e1aae6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/16528f2f6e5b4622878612520a442aff] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=48.8 K 2024-11-17T20:25:04,356 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2f394e5cb85459c890d7491c8098011, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1bd04fb5d98c445e9b5172c3c98d247e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/050f57cf985a48178360a8b6c510de08, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/160fa7a0e03b4b03a4104228d77fd66a] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=51.2 K 2024-11-17T20:25:04,356 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f24e5b9c67d34b4ebc4374ea77036559, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875099722 2024-11-17T20:25:04,356 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2f394e5cb85459c890d7491c8098011, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875099722 2024-11-17T20:25:04,356 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bcad5ea153474f349baa27fc373d0752, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731875099753 2024-11-17T20:25:04,356 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bd04fb5d98c445e9b5172c3c98d247e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731875099753 2024-11-17T20:25:04,357 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 534594f837c743fcbf086c2c16e1aae6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731875100903 2024-11-17T20:25:04,357 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050f57cf985a48178360a8b6c510de08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731875100903 2024-11-17T20:25:04,357 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 16528f2f6e5b4622878612520a442aff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731875103075 2024-11-17T20:25:04,357 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 160fa7a0e03b4b03a4104228d77fd66a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731875103075 2024-11-17T20:25:04,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-17T20:25:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,361 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:04,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/0dddc08f94a7434d984845becc00779e is 50, key is test_row_0/A:col10/1731875103118/Put/seqid=0 2024-11-17T20:25:04,375 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:04,375 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:04,375 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/de9b957669ee4640b6b89962055c3512 is 50, key is test_row_0/B:col10/1731875103075/Put/seqid=0 2024-11-17T20:25:04,376 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ec65429f69194221a886bd25a7f3d360 is 50, key is test_row_0/A:col10/1731875103075/Put/seqid=0 2024-11-17T20:25:04,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742146_1322 (size=13187) 2024-11-17T20:25:04,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742144_1320 (size=12301) 2024-11-17T20:25:04,392 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/0dddc08f94a7434d984845becc00779e 2024-11-17T20:25:04,392 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/de9b957669ee4640b6b89962055c3512 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/de9b957669ee4640b6b89962055c3512 2024-11-17T20:25:04,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742145_1321 (size=13187) 2024-11-17T20:25:04,398 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into de9b957669ee4640b6b89962055c3512(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:04,398 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:04,398 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=12, startTime=1731875104354; duration=0sec 2024-11-17T20:25:04,399 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:04,399 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:25:04,399 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:04,400 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:04,400 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:25:04,400 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,400 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3d9c2bb4f6904095b525bae6ae1c96fe, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/bc58536da68c4121a2fce60edc7d4beb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85f64025ca0240e6964332776f68efd5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e93fedcf33984fd888de1bf14c3332d4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=48.8 K 2024-11-17T20:25:04,400 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9c2bb4f6904095b525bae6ae1c96fe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731875099722 2024-11-17T20:25:04,401 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bc58536da68c4121a2fce60edc7d4beb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731875099753 2024-11-17T20:25:04,401 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f64025ca0240e6964332776f68efd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731875100903 2024-11-17T20:25:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/51c43db4eab248f89118ebde8ba9aa02 is 50, key is test_row_0/B:col10/1731875103118/Put/seqid=0 2024-11-17T20:25:04,403 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e93fedcf33984fd888de1bf14c3332d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731875103075 2024-11-17T20:25:04,415 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:04,415 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/d50fa82ddf7b4451906afd4f9e009b18 is 50, key is test_row_0/C:col10/1731875103075/Put/seqid=0 2024-11-17T20:25:04,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742148_1324 (size=13153) 2024-11-17T20:25:04,454 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/d50fa82ddf7b4451906afd4f9e009b18 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d50fa82ddf7b4451906afd4f9e009b18 2024-11-17T20:25:04,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742147_1323 (size=12301) 2024-11-17T20:25:04,458 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/51c43db4eab248f89118ebde8ba9aa02 2024-11-17T20:25:04,459 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into d50fa82ddf7b4451906afd4f9e009b18(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:04,459 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:04,459 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=12, startTime=1731875104355; duration=0sec 2024-11-17T20:25:04,459 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:04,459 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:25:04,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/014ede5eafdf408189c2c8ed42041788 is 50, key is test_row_0/C:col10/1731875103118/Put/seqid=0 2024-11-17T20:25:04,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742149_1325 (size=12301) 2024-11-17T20:25:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T20:25:04,804 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/ec65429f69194221a886bd25a7f3d360 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ec65429f69194221a886bd25a7f3d360 2024-11-17T20:25:04,827 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into ec65429f69194221a886bd25a7f3d360(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:04,829 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:04,829 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=12, startTime=1731875104354; duration=0sec 2024-11-17T20:25:04,830 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:04,830 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:25:04,873 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/014ede5eafdf408189c2c8ed42041788 2024-11-17T20:25:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/0dddc08f94a7434d984845becc00779e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/0dddc08f94a7434d984845becc00779e 2024-11-17T20:25:04,892 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/0dddc08f94a7434d984845becc00779e, entries=150, sequenceid=389, filesize=12.0 K 2024-11-17T20:25:04,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/51c43db4eab248f89118ebde8ba9aa02 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/51c43db4eab248f89118ebde8ba9aa02 2024-11-17T20:25:04,897 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/51c43db4eab248f89118ebde8ba9aa02, entries=150, sequenceid=389, filesize=12.0 K 2024-11-17T20:25:04,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/014ede5eafdf408189c2c8ed42041788 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/014ede5eafdf408189c2c8ed42041788 2024-11-17T20:25:04,902 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/014ede5eafdf408189c2c8ed42041788, entries=150, sequenceid=389, filesize=12.0 K 2024-11-17T20:25:04,903 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 542ms, sequenceid=389, compaction requested=false 2024-11-17T20:25:04,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:04,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:04,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-17T20:25:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-17T20:25:04,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T20:25:04,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-17T20:25:04,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.0100 sec 2024-11-17T20:25:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T20:25:05,040 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:05,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-17T20:25:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T20:25:05,045 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:05,045 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:05,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T20:25:05,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-17T20:25:05,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:05,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-17T20:25:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-17T20:25:05,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-17T20:25:05,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-17T20:25:05,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 162 msec 2024-11-17T20:25:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:05,278 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:05,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:25:05,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/e7a50f2ce629434389dd3dd376122f60 is 50, key is test_row_0/A:col10/1731875105277/Put/seqid=0 2024-11-17T20:25:05,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742150_1326 (size=12301) 2024-11-17T20:25:05,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/e7a50f2ce629434389dd3dd376122f60 2024-11-17T20:25:05,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/963a4305f75d4e5abf6a0eda0c6f2684 is 50, key is test_row_0/B:col10/1731875105277/Put/seqid=0 2024-11-17T20:25:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875165315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875165316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875165316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875165320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742151_1327 (size=12301) 2024-11-17T20:25:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T20:25:05,360 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:05,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees 2024-11-17T20:25:05,362 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=94, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T20:25:05,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=94, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:05,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:05,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875165430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875165430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875165430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875165430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T20:25:05,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T20:25:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875165639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875165639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875165639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875165639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T20:25:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T20:25:05,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/963a4305f75d4e5abf6a0eda0c6f2684 2024-11-17T20:25:05,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/a0468a57277f4a2fba693fc01af1cba5 is 50, key is test_row_0/C:col10/1731875105277/Put/seqid=0 2024-11-17T20:25:05,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742152_1328 (size=12301) 2024-11-17T20:25:05,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T20:25:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875165950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875165950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875165950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875165950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:05,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T20:25:05,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:05,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:05,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T20:25:06,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T20:25:06,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:06,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:06,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:06,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:06,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:06,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/a0468a57277f4a2fba693fc01af1cba5 2024-11-17T20:25:06,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/e7a50f2ce629434389dd3dd376122f60 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e7a50f2ce629434389dd3dd376122f60 2024-11-17T20:25:06,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e7a50f2ce629434389dd3dd376122f60, entries=150, sequenceid=404, filesize=12.0 K 2024-11-17T20:25:06,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/963a4305f75d4e5abf6a0eda0c6f2684 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/963a4305f75d4e5abf6a0eda0c6f2684 2024-11-17T20:25:06,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/963a4305f75d4e5abf6a0eda0c6f2684, entries=150, sequenceid=404, filesize=12.0 K 2024-11-17T20:25:06,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/a0468a57277f4a2fba693fc01af1cba5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/a0468a57277f4a2fba693fc01af1cba5 2024-11-17T20:25:06,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/a0468a57277f4a2fba693fc01af1cba5, entries=150, sequenceid=404, filesize=12.0 K 2024-11-17T20:25:06,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 875ms, sequenceid=404, compaction requested=true 2024-11-17T20:25:06,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:06,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:06,154 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:06,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:06,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:06,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:06,154 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:06,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:06,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:06,155 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:06,155 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:06,155 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:25:06,155 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:25:06,155 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:06,155 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:06,155 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ec65429f69194221a886bd25a7f3d360, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/0dddc08f94a7434d984845becc00779e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e7a50f2ce629434389dd3dd376122f60] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=36.9 K 2024-11-17T20:25:06,155 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/de9b957669ee4640b6b89962055c3512, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/51c43db4eab248f89118ebde8ba9aa02, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/963a4305f75d4e5abf6a0eda0c6f2684] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=36.9 K 2024-11-17T20:25:06,155 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting de9b957669ee4640b6b89962055c3512, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731875103075 2024-11-17T20:25:06,155 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec65429f69194221a886bd25a7f3d360, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731875103075 2024-11-17T20:25:06,156 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dddc08f94a7434d984845becc00779e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731875103118 2024-11-17T20:25:06,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c43db4eab248f89118ebde8ba9aa02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731875103118 2024-11-17T20:25:06,156 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a50f2ce629434389dd3dd376122f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731875105274 2024-11-17T20:25:06,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 963a4305f75d4e5abf6a0eda0c6f2684, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731875105274 2024-11-17T20:25:06,163 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:06,163 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:06,163 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a38c3197c98341678ec40e51ea57d756 is 50, key is test_row_0/A:col10/1731875105277/Put/seqid=0 2024-11-17T20:25:06,163 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/d9b0554270d34d2a9bc149c9421a8d1c is 50, key is test_row_0/B:col10/1731875105277/Put/seqid=0 2024-11-17T20:25:06,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742153_1329 (size=13289) 2024-11-17T20:25:06,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742154_1330 (size=13289) 2024-11-17T20:25:06,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-17T20:25:06,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:06,280 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:06,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/5e05960e39954d4f804309089cd5b646 is 50, key is test_row_0/A:col10/1731875105319/Put/seqid=0 2024-11-17T20:25:06,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742155_1331 (size=12301) 2024-11-17T20:25:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. as already flushing 2024-11-17T20:25:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875166466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875166467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875166467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875166468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T20:25:06,570 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/a38c3197c98341678ec40e51ea57d756 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a38c3197c98341678ec40e51ea57d756 2024-11-17T20:25:06,574 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into a38c3197c98341678ec40e51ea57d756(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:06,574 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:06,575 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875106153; duration=0sec 2024-11-17T20:25:06,575 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:06,575 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:25:06,575 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:06,576 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:06,576 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:25:06,576 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:06,576 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d50fa82ddf7b4451906afd4f9e009b18, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/014ede5eafdf408189c2c8ed42041788, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/a0468a57277f4a2fba693fc01af1cba5] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=36.9 K 2024-11-17T20:25:06,576 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d50fa82ddf7b4451906afd4f9e009b18, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731875103075 2024-11-17T20:25:06,577 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014ede5eafdf408189c2c8ed42041788, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731875103118 2024-11-17T20:25:06,577 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0468a57277f4a2fba693fc01af1cba5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731875105274 2024-11-17T20:25:06,580 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/d9b0554270d34d2a9bc149c9421a8d1c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d9b0554270d34d2a9bc149c9421a8d1c 2024-11-17T20:25:06,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875166579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875166579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875166580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875166580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,584 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into d9b0554270d34d2a9bc149c9421a8d1c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:06,584 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:06,584 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:06,584 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875106154; duration=0sec 2024-11-17T20:25:06,584 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:06,584 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:25:06,585 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/fe265cbe27544ef6b901fd3a17fecbb2 is 50, key is test_row_0/C:col10/1731875105277/Put/seqid=0 2024-11-17T20:25:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742156_1332 (size=13255) 2024-11-17T20:25:06,592 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/fe265cbe27544ef6b901fd3a17fecbb2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/fe265cbe27544ef6b901fd3a17fecbb2 2024-11-17T20:25:06,596 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into fe265cbe27544ef6b901fd3a17fecbb2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:06,596 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:06,596 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=13, startTime=1731875106154; duration=0sec 2024-11-17T20:25:06,596 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:06,596 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:25:06,688 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/5e05960e39954d4f804309089cd5b646 2024-11-17T20:25:06,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7c5c22cb81f44e7da94314afb5024dce is 50, key is test_row_0/B:col10/1731875105319/Put/seqid=0 2024-11-17T20:25:06,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742157_1333 (size=12301) 2024-11-17T20:25:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875166789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875166789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875166790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875166790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875167099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875167099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875167099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,102 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7c5c22cb81f44e7da94314afb5024dce 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875167099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/65ce3103b5064df7bf8fd1f1d351d8a7 is 50, key is test_row_0/C:col10/1731875105319/Put/seqid=0 2024-11-17T20:25:07,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742158_1334 (size=12301) 2024-11-17T20:25:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T20:25:07,517 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/65ce3103b5064df7bf8fd1f1d351d8a7 2024-11-17T20:25:07,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/5e05960e39954d4f804309089cd5b646 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5e05960e39954d4f804309089cd5b646 2024-11-17T20:25:07,524 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5e05960e39954d4f804309089cd5b646, entries=150, sequenceid=429, filesize=12.0 K 2024-11-17T20:25:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7c5c22cb81f44e7da94314afb5024dce as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7c5c22cb81f44e7da94314afb5024dce 2024-11-17T20:25:07,528 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7c5c22cb81f44e7da94314afb5024dce, entries=150, sequenceid=429, filesize=12.0 K 2024-11-17T20:25:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/65ce3103b5064df7bf8fd1f1d351d8a7 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/65ce3103b5064df7bf8fd1f1d351d8a7 2024-11-17T20:25:07,531 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/65ce3103b5064df7bf8fd1f1d351d8a7, entries=150, sequenceid=429, filesize=12.0 K 2024-11-17T20:25:07,532 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1253ms, sequenceid=429, compaction requested=false 2024-11-17T20:25:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=95 2024-11-17T20:25:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=95 2024-11-17T20:25:07,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-17T20:25:07,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-17T20:25:07,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees in 2.1750 sec 2024-11-17T20:25:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:07,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:07,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:07,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/5d073f28a42747b993e935edb203b227 is 50, key is test_row_0/A:col10/1731875106465/Put/seqid=0 2024-11-17T20:25:07,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742159_1335 (size=17181) 2024-11-17T20:25:07,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/5d073f28a42747b993e935edb203b227 2024-11-17T20:25:07,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/2652460480dd46eb8251ae952c6b6f5c is 50, key is test_row_0/B:col10/1731875106465/Put/seqid=0 2024-11-17T20:25:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875167635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875167636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875167637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875167639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742160_1336 (size=12301) 2024-11-17T20:25:07,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875167749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875167750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875167750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875167750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875167959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875167959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875167959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875167960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:08,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/2652460480dd46eb8251ae952c6b6f5c 2024-11-17T20:25:08,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/0c022b002bcc4f4982908c1c646e9623 is 50, key is test_row_0/C:col10/1731875106465/Put/seqid=0 2024-11-17T20:25:08,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742161_1337 (size=12301) 2024-11-17T20:25:08,104 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-17T20:25:08,104 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,104 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,104 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,105 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-17T20:25:08,105 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,105 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,105 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,107 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-17T20:25:08,107 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-17T20:25:08,107 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,107 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,107 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,107 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,108 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,108 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,108 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-17T20:25:08,108 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,108 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,108 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:08,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:08,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39022 deadline: 1731875168270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39000 deadline: 1731875168270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38994 deadline: 1731875168270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:08,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1731875168270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:08,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/0c022b002bcc4f4982908c1c646e9623 2024-11-17T20:25:08,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/5d073f28a42747b993e935edb203b227 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5d073f28a42747b993e935edb203b227 2024-11-17T20:25:08,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5d073f28a42747b993e935edb203b227, entries=250, sequenceid=445, filesize=16.8 K 2024-11-17T20:25:08,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/2652460480dd46eb8251ae952c6b6f5c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/2652460480dd46eb8251ae952c6b6f5c 2024-11-17T20:25:08,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/2652460480dd46eb8251ae952c6b6f5c, entries=150, sequenceid=445, filesize=12.0 K 2024-11-17T20:25:08,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/0c022b002bcc4f4982908c1c646e9623 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/0c022b002bcc4f4982908c1c646e9623 2024-11-17T20:25:08,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/0c022b002bcc4f4982908c1c646e9623, entries=150, sequenceid=445, filesize=12.0 K 2024-11-17T20:25:08,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 881ms, sequenceid=445, compaction requested=true 2024-11-17T20:25:08,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:08,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:08,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:08,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:08,492 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:08,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:08,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 31e1f74c01abc6e3eb193ea5bbd4a22a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:08,492 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:08,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:08,493 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:08,493 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:08,493 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/B is initiating minor compaction (all files) 2024-11-17T20:25:08,493 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/A is initiating minor compaction (all files) 2024-11-17T20:25:08,494 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/B in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:08,494 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/A in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:08,494 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d9b0554270d34d2a9bc149c9421a8d1c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7c5c22cb81f44e7da94314afb5024dce, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/2652460480dd46eb8251ae952c6b6f5c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=37.0 K 2024-11-17T20:25:08,494 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a38c3197c98341678ec40e51ea57d756, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5e05960e39954d4f804309089cd5b646, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5d073f28a42747b993e935edb203b227] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=41.8 K 2024-11-17T20:25:08,494 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38c3197c98341678ec40e51ea57d756, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731875105274 2024-11-17T20:25:08,494 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b0554270d34d2a9bc149c9421a8d1c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731875105274 2024-11-17T20:25:08,494 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c5c22cb81f44e7da94314afb5024dce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731875105293 2024-11-17T20:25:08,494 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e05960e39954d4f804309089cd5b646, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731875105293 2024-11-17T20:25:08,495 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2652460480dd46eb8251ae952c6b6f5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731875106465 2024-11-17T20:25:08,495 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d073f28a42747b993e935edb203b227, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731875106465 2024-11-17T20:25:08,505 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#B#compaction#289 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:08,506 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/3e907de5b7f54c5da5f1102a6aa2703c is 50, key is test_row_0/B:col10/1731875106465/Put/seqid=0 2024-11-17T20:25:08,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742162_1338 (size=13391) 2024-11-17T20:25:08,530 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#A#compaction#290 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:08,531 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/cacc3b66c1bd4623a276a70fec5473f2 is 50, key is test_row_0/A:col10/1731875106465/Put/seqid=0 2024-11-17T20:25:08,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742163_1339 (size=13391) 2024-11-17T20:25:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:08,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:08,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:25:08,781 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-17T20:25:08,781 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,781 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,781 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-17T20:25:08,782 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,782 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,782 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-17T20:25:08,782 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,782 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,782 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,782 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,782 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,784 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-17T20:25:08,784 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:08,784 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,784 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:08,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:08,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/91ba1c2d962d46e7a01384dd9c03ca64 is 50, key is test_row_0/A:col10/1731875107635/Put/seqid=0 2024-11-17T20:25:08,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742164_1340 (size=12301) 2024-11-17T20:25:08,924 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/3e907de5b7f54c5da5f1102a6aa2703c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/3e907de5b7f54c5da5f1102a6aa2703c 2024-11-17T20:25:08,930 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/B of 31e1f74c01abc6e3eb193ea5bbd4a22a into 3e907de5b7f54c5da5f1102a6aa2703c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:08,930 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:08,930 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/B, priority=13, startTime=1731875108492; duration=0sec 2024-11-17T20:25:08,931 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:08,931 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:B 2024-11-17T20:25:08,931 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:08,932 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:08,932 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 31e1f74c01abc6e3eb193ea5bbd4a22a/C is initiating minor compaction (all files) 2024-11-17T20:25:08,932 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 31e1f74c01abc6e3eb193ea5bbd4a22a/C in TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:08,932 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/fe265cbe27544ef6b901fd3a17fecbb2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/65ce3103b5064df7bf8fd1f1d351d8a7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/0c022b002bcc4f4982908c1c646e9623] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp, totalSize=37.0 K 2024-11-17T20:25:08,933 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting fe265cbe27544ef6b901fd3a17fecbb2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731875105274 2024-11-17T20:25:08,933 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ce3103b5064df7bf8fd1f1d351d8a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731875105293 2024-11-17T20:25:08,933 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c022b002bcc4f4982908c1c646e9623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731875106465 2024-11-17T20:25:08,939 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/cacc3b66c1bd4623a276a70fec5473f2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/cacc3b66c1bd4623a276a70fec5473f2 2024-11-17T20:25:08,942 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 31e1f74c01abc6e3eb193ea5bbd4a22a#C#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:08,943 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/ee049392f4f14180b2cafc7a8f58aa1d is 50, key is test_row_0/C:col10/1731875106465/Put/seqid=0 2024-11-17T20:25:08,943 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/A of 31e1f74c01abc6e3eb193ea5bbd4a22a into cacc3b66c1bd4623a276a70fec5473f2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:08,943 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:08,943 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/A, priority=13, startTime=1731875108492; duration=0sec 2024-11-17T20:25:08,943 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:08,943 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:A 2024-11-17T20:25:08,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742165_1341 (size=13357) 2024-11-17T20:25:09,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/91ba1c2d962d46e7a01384dd9c03ca64 2024-11-17T20:25:09,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7b02865d9e694a2ca649c3e4c956c380 is 50, key is test_row_0/B:col10/1731875107635/Put/seqid=0 2024-11-17T20:25:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742166_1342 (size=12301) 2024-11-17T20:25:09,356 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/ee049392f4f14180b2cafc7a8f58aa1d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/ee049392f4f14180b2cafc7a8f58aa1d 2024-11-17T20:25:09,363 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 31e1f74c01abc6e3eb193ea5bbd4a22a/C of 31e1f74c01abc6e3eb193ea5bbd4a22a into ee049392f4f14180b2cafc7a8f58aa1d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:09,363 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:09,363 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., storeName=31e1f74c01abc6e3eb193ea5bbd4a22a/C, priority=13, startTime=1731875108492; duration=0sec 2024-11-17T20:25:09,363 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:09,363 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 31e1f74c01abc6e3eb193ea5bbd4a22a:C 2024-11-17T20:25:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T20:25:09,520 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:09,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7b02865d9e694a2ca649c3e4c956c380 2024-11-17T20:25:09,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/f6269f03bab9408189862972210d9976 is 50, key is test_row_0/C:col10/1731875107635/Put/seqid=0 2024-11-17T20:25:09,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742167_1343 (size=12301) 2024-11-17T20:25:09,831 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-17T20:25:09,831 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:09,831 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:09,832 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:09,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4138 2024-11-17T20:25:09,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3950 2024-11-17T20:25:09,833 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3975 2024-11-17T20:25:09,833 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4158 2024-11-17T20:25:09,833 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3940 2024-11-17T20:25:09,833 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T20:25:09,833 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:25:09,833 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:25:09,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:09,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:09,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:09,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T20:25:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T20:25:09,842 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875109841"}]},"ts":"1731875109841"} 2024-11-17T20:25:09,844 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T20:25:09,844 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T20:25:09,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:25:09,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, UNASSIGN}] 2024-11-17T20:25:09,848 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, UNASSIGN 2024-11-17T20:25:09,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=31e1f74c01abc6e3eb193ea5bbd4a22a, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:09,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, UNASSIGN because future has completed 2024-11-17T20:25:09,851 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:25:09,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:25:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T20:25:10,004 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:10,004 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:25:10,004 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 31e1f74c01abc6e3eb193ea5bbd4a22a, disabling compactions & flushes 2024-11-17T20:25:10,005 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:10,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/f6269f03bab9408189862972210d9976 2024-11-17T20:25:10,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/91ba1c2d962d46e7a01384dd9c03ca64 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/91ba1c2d962d46e7a01384dd9c03ca64 2024-11-17T20:25:10,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/91ba1c2d962d46e7a01384dd9c03ca64, entries=150, sequenceid=471, filesize=12.0 K 2024-11-17T20:25:10,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/7b02865d9e694a2ca649c3e4c956c380 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7b02865d9e694a2ca649c3e4c956c380 2024-11-17T20:25:10,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7b02865d9e694a2ca649c3e4c956c380, entries=150, sequenceid=471, filesize=12.0 K 2024-11-17T20:25:10,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/f6269f03bab9408189862972210d9976 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/f6269f03bab9408189862972210d9976 2024-11-17T20:25:10,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/f6269f03bab9408189862972210d9976, entries=150, sequenceid=471, filesize=12.0 K 2024-11-17T20:25:10,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=13.42 KB/13740 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1257ms, sequenceid=471, compaction requested=false 2024-11-17T20:25:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: 2024-11-17T20:25:10,038 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. after waiting 0 ms 2024-11-17T20:25:10,038 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:10,038 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(2902): Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-17T20:25:10,041 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/d8eedc737be34976b8e81e11c65f5128 is 50, key is test_row_2/A:col10/1731875109829/Put/seqid=0 2024-11-17T20:25:10,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742168_1344 (size=7415) 2024-11-17T20:25:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T20:25:10,445 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/d8eedc737be34976b8e81e11c65f5128 2024-11-17T20:25:10,458 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/ecae93dff58a479d9de5d8852a838961 is 50, key is test_row_2/B:col10/1731875109829/Put/seqid=0 2024-11-17T20:25:10,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742169_1345 (size=7415) 2024-11-17T20:25:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T20:25:10,862 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/ecae93dff58a479d9de5d8852a838961 2024-11-17T20:25:10,869 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/56d74618ddea4f3692c826fc3bb96958 is 50, key is test_row_2/C:col10/1731875109829/Put/seqid=0 2024-11-17T20:25:10,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742170_1346 (size=7415) 2024-11-17T20:25:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T20:25:11,273 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/56d74618ddea4f3692c826fc3bb96958 2024-11-17T20:25:11,276 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/A/d8eedc737be34976b8e81e11c65f5128 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d8eedc737be34976b8e81e11c65f5128 2024-11-17T20:25:11,280 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d8eedc737be34976b8e81e11c65f5128, entries=50, sequenceid=479, filesize=7.2 K 2024-11-17T20:25:11,281 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/B/ecae93dff58a479d9de5d8852a838961 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ecae93dff58a479d9de5d8852a838961 2024-11-17T20:25:11,284 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ecae93dff58a479d9de5d8852a838961, entries=50, sequenceid=479, filesize=7.2 K 2024-11-17T20:25:11,285 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/.tmp/C/56d74618ddea4f3692c826fc3bb96958 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/56d74618ddea4f3692c826fc3bb96958 2024-11-17T20:25:11,289 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/56d74618ddea4f3692c826fc3bb96958, entries=50, sequenceid=479, filesize=7.2 K 2024-11-17T20:25:11,290 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1252ms, sequenceid=479, compaction requested=true 2024-11-17T20:25:11,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/692365f3ee2d44ba91fc4efc148604fd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ba522a46bd64467ca9ed2706fd0f3f32, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed86c5ceeab642a0b03b304b2c387a13, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed1e4a1331be442d89bca112f26c54db, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a18d268fec4d4d309b7a8f1cfe5c12b0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/8b8dbe4c9f1b4152b07b73406a5743d8, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32f6474d4c0a4baf9f2a3baa29d29d70, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1efcc9176eff447f81ed305a636677f3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32b43282ff264088bca3166afd93d16b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4c81b903d7c84cb786fc93aa1b6c50d2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a9a039908f694ad58a775a1194efaafd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/6e6ca638d66f441798e170fc8c022edb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/00ef5e2d630049e3acbe968085ad4357, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2ae2b4416e94a139be32f473a9e4b24, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/40988e201481440c83cecb44015da4cc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e44fabbb23c949f6bbe05c9718eb9fb3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d5c49bbcee1d4f809b4fad9dadfdb825, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/49f89769da9942d1adceef8c6846cc88, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/b7404944950c4cce80fcf7c45b3d0a42, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a64e0ecc4449492180ed8c89e3b3a74f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4ea6d30865454882824c55393cd95556, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/38bd24066603479199ab39ae89dbf651, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2f394e5cb85459c890d7491c8098011, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1bd04fb5d98c445e9b5172c3c98d247e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/050f57cf985a48178360a8b6c510de08, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/160fa7a0e03b4b03a4104228d77fd66a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ec65429f69194221a886bd25a7f3d360, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/0dddc08f94a7434d984845becc00779e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a38c3197c98341678ec40e51ea57d756, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e7a50f2ce629434389dd3dd376122f60, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5e05960e39954d4f804309089cd5b646, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5d073f28a42747b993e935edb203b227] to archive 2024-11-17T20:25:11,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:11,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/692365f3ee2d44ba91fc4efc148604fd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/692365f3ee2d44ba91fc4efc148604fd 2024-11-17T20:25:11,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ba522a46bd64467ca9ed2706fd0f3f32 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ba522a46bd64467ca9ed2706fd0f3f32 2024-11-17T20:25:11,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed86c5ceeab642a0b03b304b2c387a13 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed86c5ceeab642a0b03b304b2c387a13 2024-11-17T20:25:11,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed1e4a1331be442d89bca112f26c54db to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ed1e4a1331be442d89bca112f26c54db 2024-11-17T20:25:11,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a18d268fec4d4d309b7a8f1cfe5c12b0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a18d268fec4d4d309b7a8f1cfe5c12b0 2024-11-17T20:25:11,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/8b8dbe4c9f1b4152b07b73406a5743d8 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/8b8dbe4c9f1b4152b07b73406a5743d8 2024-11-17T20:25:11,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32f6474d4c0a4baf9f2a3baa29d29d70 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32f6474d4c0a4baf9f2a3baa29d29d70 2024-11-17T20:25:11,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1efcc9176eff447f81ed305a636677f3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1efcc9176eff447f81ed305a636677f3 2024-11-17T20:25:11,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32b43282ff264088bca3166afd93d16b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/32b43282ff264088bca3166afd93d16b 2024-11-17T20:25:11,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4c81b903d7c84cb786fc93aa1b6c50d2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4c81b903d7c84cb786fc93aa1b6c50d2 2024-11-17T20:25:11,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a9a039908f694ad58a775a1194efaafd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a9a039908f694ad58a775a1194efaafd 2024-11-17T20:25:11,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/6e6ca638d66f441798e170fc8c022edb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/6e6ca638d66f441798e170fc8c022edb 2024-11-17T20:25:11,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/00ef5e2d630049e3acbe968085ad4357 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/00ef5e2d630049e3acbe968085ad4357 2024-11-17T20:25:11,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2ae2b4416e94a139be32f473a9e4b24 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2ae2b4416e94a139be32f473a9e4b24 2024-11-17T20:25:11,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/40988e201481440c83cecb44015da4cc to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/40988e201481440c83cecb44015da4cc 2024-11-17T20:25:11,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e44fabbb23c949f6bbe05c9718eb9fb3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e44fabbb23c949f6bbe05c9718eb9fb3 2024-11-17T20:25:11,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d5c49bbcee1d4f809b4fad9dadfdb825 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d5c49bbcee1d4f809b4fad9dadfdb825 2024-11-17T20:25:11,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/49f89769da9942d1adceef8c6846cc88 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/49f89769da9942d1adceef8c6846cc88 2024-11-17T20:25:11,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/b7404944950c4cce80fcf7c45b3d0a42 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/b7404944950c4cce80fcf7c45b3d0a42 2024-11-17T20:25:11,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a64e0ecc4449492180ed8c89e3b3a74f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a64e0ecc4449492180ed8c89e3b3a74f 2024-11-17T20:25:11,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4ea6d30865454882824c55393cd95556 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/4ea6d30865454882824c55393cd95556 2024-11-17T20:25:11,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/38bd24066603479199ab39ae89dbf651 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/38bd24066603479199ab39ae89dbf651 2024-11-17T20:25:11,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2f394e5cb85459c890d7491c8098011 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/c2f394e5cb85459c890d7491c8098011 2024-11-17T20:25:11,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1bd04fb5d98c445e9b5172c3c98d247e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/1bd04fb5d98c445e9b5172c3c98d247e 2024-11-17T20:25:11,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/050f57cf985a48178360a8b6c510de08 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/050f57cf985a48178360a8b6c510de08 2024-11-17T20:25:11,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/160fa7a0e03b4b03a4104228d77fd66a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/160fa7a0e03b4b03a4104228d77fd66a 2024-11-17T20:25:11,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ec65429f69194221a886bd25a7f3d360 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/ec65429f69194221a886bd25a7f3d360 2024-11-17T20:25:11,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/0dddc08f94a7434d984845becc00779e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/0dddc08f94a7434d984845becc00779e 2024-11-17T20:25:11,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a38c3197c98341678ec40e51ea57d756 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/a38c3197c98341678ec40e51ea57d756 2024-11-17T20:25:11,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e7a50f2ce629434389dd3dd376122f60 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/e7a50f2ce629434389dd3dd376122f60 2024-11-17T20:25:11,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5e05960e39954d4f804309089cd5b646 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5e05960e39954d4f804309089cd5b646 2024-11-17T20:25:11,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5d073f28a42747b993e935edb203b227 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/5d073f28a42747b993e935edb203b227 2024-11-17T20:25:11,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/0f9c3432d6dd4ef1aaa06510b3e9a34e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/c006b73842ae4042ade63c0bbce1f084, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d6702ac017c64cdfb4576a916f6da5ad, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7d6d3c7987d84adfa7806e15fe676c9e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/1ac63d9bf1dc41c0ab449c42b10eb33a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bd267cdc7cd84121a8b6c2eac13257ae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/50c9445169764d9585819246295c13b3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bf008f5eef62417fa0057828b58e429c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e4304c0c7ed94bc9967c6e255e9313bc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/5077b4b3a886439b93bf3cb75e411074, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e5d77edb619a4df28efb93e623bb706b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/79be3520f3574f6a8ae380d32f73d04d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/89f749ffc6d6408f9dc27996bf05b0b7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/82265ffb24f44797b8db06aa3c990629, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/557139d0296547daaeb885470a2a7191, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/6b8d4877cba74894aa6fbef2078190b9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f5e7be1ec025465ea0a9db0430ae3652, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/20164bb959d74454a8f3be4e5656ac47, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/359fabe140394a14a935598ee850009a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/078880e5da42481d94bf9a84cfe427ba, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f24e5b9c67d34b4ebc4374ea77036559, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/521850e5a0944979a0d2971da050b0f1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bcad5ea153474f349baa27fc373d0752, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/534594f837c743fcbf086c2c16e1aae6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/de9b957669ee4640b6b89962055c3512, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/16528f2f6e5b4622878612520a442aff, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/51c43db4eab248f89118ebde8ba9aa02, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d9b0554270d34d2a9bc149c9421a8d1c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/963a4305f75d4e5abf6a0eda0c6f2684, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7c5c22cb81f44e7da94314afb5024dce, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/2652460480dd46eb8251ae952c6b6f5c] to archive 2024-11-17T20:25:11,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:11,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/0f9c3432d6dd4ef1aaa06510b3e9a34e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/0f9c3432d6dd4ef1aaa06510b3e9a34e 2024-11-17T20:25:11,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/c006b73842ae4042ade63c0bbce1f084 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/c006b73842ae4042ade63c0bbce1f084 2024-11-17T20:25:11,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d6702ac017c64cdfb4576a916f6da5ad to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d6702ac017c64cdfb4576a916f6da5ad 2024-11-17T20:25:11,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7d6d3c7987d84adfa7806e15fe676c9e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7d6d3c7987d84adfa7806e15fe676c9e 2024-11-17T20:25:11,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/1ac63d9bf1dc41c0ab449c42b10eb33a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/1ac63d9bf1dc41c0ab449c42b10eb33a 2024-11-17T20:25:11,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bd267cdc7cd84121a8b6c2eac13257ae to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bd267cdc7cd84121a8b6c2eac13257ae 2024-11-17T20:25:11,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/50c9445169764d9585819246295c13b3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/50c9445169764d9585819246295c13b3 2024-11-17T20:25:11,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bf008f5eef62417fa0057828b58e429c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bf008f5eef62417fa0057828b58e429c 2024-11-17T20:25:11,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e4304c0c7ed94bc9967c6e255e9313bc to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e4304c0c7ed94bc9967c6e255e9313bc 2024-11-17T20:25:11,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/5077b4b3a886439b93bf3cb75e411074 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/5077b4b3a886439b93bf3cb75e411074 2024-11-17T20:25:11,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e5d77edb619a4df28efb93e623bb706b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/e5d77edb619a4df28efb93e623bb706b 2024-11-17T20:25:11,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/79be3520f3574f6a8ae380d32f73d04d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/79be3520f3574f6a8ae380d32f73d04d 2024-11-17T20:25:11,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/89f749ffc6d6408f9dc27996bf05b0b7 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/89f749ffc6d6408f9dc27996bf05b0b7 2024-11-17T20:25:11,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/82265ffb24f44797b8db06aa3c990629 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/82265ffb24f44797b8db06aa3c990629 2024-11-17T20:25:11,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/557139d0296547daaeb885470a2a7191 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/557139d0296547daaeb885470a2a7191 2024-11-17T20:25:11,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/6b8d4877cba74894aa6fbef2078190b9 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/6b8d4877cba74894aa6fbef2078190b9 2024-11-17T20:25:11,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ce9f9ee77d4047b6a61a3ab7a7b6e2d5 2024-11-17T20:25:11,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f5e7be1ec025465ea0a9db0430ae3652 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f5e7be1ec025465ea0a9db0430ae3652 2024-11-17T20:25:11,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/20164bb959d74454a8f3be4e5656ac47 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/20164bb959d74454a8f3be4e5656ac47 2024-11-17T20:25:11,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/359fabe140394a14a935598ee850009a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/359fabe140394a14a935598ee850009a 2024-11-17T20:25:11,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/078880e5da42481d94bf9a84cfe427ba to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/078880e5da42481d94bf9a84cfe427ba 2024-11-17T20:25:11,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f24e5b9c67d34b4ebc4374ea77036559 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/f24e5b9c67d34b4ebc4374ea77036559 2024-11-17T20:25:11,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/521850e5a0944979a0d2971da050b0f1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/521850e5a0944979a0d2971da050b0f1 2024-11-17T20:25:11,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bcad5ea153474f349baa27fc373d0752 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/bcad5ea153474f349baa27fc373d0752 2024-11-17T20:25:11,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/534594f837c743fcbf086c2c16e1aae6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/534594f837c743fcbf086c2c16e1aae6 2024-11-17T20:25:11,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/de9b957669ee4640b6b89962055c3512 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/de9b957669ee4640b6b89962055c3512 2024-11-17T20:25:11,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/16528f2f6e5b4622878612520a442aff to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/16528f2f6e5b4622878612520a442aff 2024-11-17T20:25:11,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/51c43db4eab248f89118ebde8ba9aa02 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/51c43db4eab248f89118ebde8ba9aa02 2024-11-17T20:25:11,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d9b0554270d34d2a9bc149c9421a8d1c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/d9b0554270d34d2a9bc149c9421a8d1c 2024-11-17T20:25:11,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/963a4305f75d4e5abf6a0eda0c6f2684 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/963a4305f75d4e5abf6a0eda0c6f2684 2024-11-17T20:25:11,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7c5c22cb81f44e7da94314afb5024dce to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7c5c22cb81f44e7da94314afb5024dce 2024-11-17T20:25:11,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/2652460480dd46eb8251ae952c6b6f5c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/2652460480dd46eb8251ae952c6b6f5c 2024-11-17T20:25:11,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c298a232958c42608fa4c228724b38cd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85921f50ef9745cda4c9dd4503dbee3b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1148a2de4bdd4a39963a753ffbcab5f0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/11141fd2053b464d972f3786cf9dc505, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c26feba91acd40129c2bd4d891472389, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/94b4c35f698343fd880f61368de586e2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/137d3fdc6f7548f7882cafdd0d552385, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/60963ecef1b7425ca23c7096955ba826, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3aaeda73db50474fb328ee7b88986eb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1f1b842278cd4ccfb5255ff5566852cb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/53b67549518a48b4aa510ea8554b1b67, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c7bf463c1cc34914b4340dd27c376e82, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/12b3e2935ff64adebc817c5c1d7dc3d6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d7d3f7249c824e7ea1c6123d08a306f7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/44e18ce6ccb8430c9977587c23588016, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/86c837ae53cf44128db7d0584c00ddec, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/082f34f23afb4187ba7206aa5d0f68c6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/762142de797648c08485432a08b92d94, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3e01c49e1c0f4c408e15c34eaf2ebabd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/5a71df34e8bb44b09b1e48373386f4c6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3d9c2bb4f6904095b525bae6ae1c96fe, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e0e62ec1180346bfa7333970e8d15a36, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/bc58536da68c4121a2fce60edc7d4beb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85f64025ca0240e6964332776f68efd5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d50fa82ddf7b4451906afd4f9e009b18, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e93fedcf33984fd888de1bf14c3332d4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/014ede5eafdf408189c2c8ed42041788, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/fe265cbe27544ef6b901fd3a17fecbb2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/a0468a57277f4a2fba693fc01af1cba5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/65ce3103b5064df7bf8fd1f1d351d8a7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/0c022b002bcc4f4982908c1c646e9623] to archive 2024-11-17T20:25:11,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:11,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c298a232958c42608fa4c228724b38cd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c298a232958c42608fa4c228724b38cd 2024-11-17T20:25:11,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85921f50ef9745cda4c9dd4503dbee3b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85921f50ef9745cda4c9dd4503dbee3b 2024-11-17T20:25:11,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1148a2de4bdd4a39963a753ffbcab5f0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1148a2de4bdd4a39963a753ffbcab5f0 2024-11-17T20:25:11,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/11141fd2053b464d972f3786cf9dc505 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/11141fd2053b464d972f3786cf9dc505 2024-11-17T20:25:11,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c26feba91acd40129c2bd4d891472389 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c26feba91acd40129c2bd4d891472389 2024-11-17T20:25:11,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/94b4c35f698343fd880f61368de586e2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/94b4c35f698343fd880f61368de586e2 2024-11-17T20:25:11,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/137d3fdc6f7548f7882cafdd0d552385 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/137d3fdc6f7548f7882cafdd0d552385 2024-11-17T20:25:11,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/60963ecef1b7425ca23c7096955ba826 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/60963ecef1b7425ca23c7096955ba826 2024-11-17T20:25:11,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3aaeda73db50474fb328ee7b88986eb5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3aaeda73db50474fb328ee7b88986eb5 2024-11-17T20:25:11,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1f1b842278cd4ccfb5255ff5566852cb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/1f1b842278cd4ccfb5255ff5566852cb 2024-11-17T20:25:11,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/53b67549518a48b4aa510ea8554b1b67 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/53b67549518a48b4aa510ea8554b1b67 2024-11-17T20:25:11,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c7bf463c1cc34914b4340dd27c376e82 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/c7bf463c1cc34914b4340dd27c376e82 2024-11-17T20:25:11,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/12b3e2935ff64adebc817c5c1d7dc3d6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/12b3e2935ff64adebc817c5c1d7dc3d6 2024-11-17T20:25:11,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d7d3f7249c824e7ea1c6123d08a306f7 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d7d3f7249c824e7ea1c6123d08a306f7 2024-11-17T20:25:11,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/44e18ce6ccb8430c9977587c23588016 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/44e18ce6ccb8430c9977587c23588016 2024-11-17T20:25:11,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/86c837ae53cf44128db7d0584c00ddec to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/86c837ae53cf44128db7d0584c00ddec 2024-11-17T20:25:11,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/082f34f23afb4187ba7206aa5d0f68c6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/082f34f23afb4187ba7206aa5d0f68c6 2024-11-17T20:25:11,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/762142de797648c08485432a08b92d94 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/762142de797648c08485432a08b92d94 2024-11-17T20:25:11,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3e01c49e1c0f4c408e15c34eaf2ebabd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3e01c49e1c0f4c408e15c34eaf2ebabd 2024-11-17T20:25:11,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/5a71df34e8bb44b09b1e48373386f4c6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/5a71df34e8bb44b09b1e48373386f4c6 2024-11-17T20:25:11,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3d9c2bb4f6904095b525bae6ae1c96fe to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/3d9c2bb4f6904095b525bae6ae1c96fe 2024-11-17T20:25:11,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e0e62ec1180346bfa7333970e8d15a36 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e0e62ec1180346bfa7333970e8d15a36 2024-11-17T20:25:11,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/bc58536da68c4121a2fce60edc7d4beb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/bc58536da68c4121a2fce60edc7d4beb 2024-11-17T20:25:11,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85f64025ca0240e6964332776f68efd5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/85f64025ca0240e6964332776f68efd5 2024-11-17T20:25:11,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d50fa82ddf7b4451906afd4f9e009b18 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/d50fa82ddf7b4451906afd4f9e009b18 2024-11-17T20:25:11,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e93fedcf33984fd888de1bf14c3332d4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/e93fedcf33984fd888de1bf14c3332d4 2024-11-17T20:25:11,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/014ede5eafdf408189c2c8ed42041788 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/014ede5eafdf408189c2c8ed42041788 2024-11-17T20:25:11,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/fe265cbe27544ef6b901fd3a17fecbb2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/fe265cbe27544ef6b901fd3a17fecbb2 2024-11-17T20:25:11,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/a0468a57277f4a2fba693fc01af1cba5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/a0468a57277f4a2fba693fc01af1cba5 2024-11-17T20:25:11,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/65ce3103b5064df7bf8fd1f1d351d8a7 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/65ce3103b5064df7bf8fd1f1d351d8a7 2024-11-17T20:25:11,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/0c022b002bcc4f4982908c1c646e9623 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/0c022b002bcc4f4982908c1c646e9623 2024-11-17T20:25:11,378 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/recovered.edits/482.seqid, newMaxSeqId=482, maxSeqId=1 2024-11-17T20:25:11,379 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. 2024-11-17T20:25:11,379 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 31e1f74c01abc6e3eb193ea5bbd4a22a: Waiting for close lock at 1731875110004Running coprocessor pre-close hooks at 1731875110004Disabling compacts and flushes for region at 1731875110004Disabling writes for close at 1731875110038 (+34 ms)Obtaining lock to block concurrent updates at 1731875110038Preparing flush snapshotting stores in 31e1f74c01abc6e3eb193ea5bbd4a22a at 1731875110038Finished memstore snapshotting TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731875110039 (+1 ms)Flushing stores of TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a. at 1731875110039Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/A: creating writer at 1731875110039Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/A: appending metadata at 1731875110041 (+2 ms)Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/A: closing flushed file at 1731875110041Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/B: creating writer at 1731875110453 (+412 ms)Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/B: appending metadata at 1731875110457 (+4 ms)Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/B: closing flushed file at 1731875110457Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/C: creating writer at 1731875110866 (+409 ms)Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/C: appending metadata at 1731875110868 (+2 ms)Flushing 31e1f74c01abc6e3eb193ea5bbd4a22a/C: closing flushed file at 1731875110868Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c2b6707: reopening flushed file at 1731875111276 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fc637ce: reopening flushed file at 1731875111280 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47e23850: reopening flushed file at 1731875111284 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 31e1f74c01abc6e3eb193ea5bbd4a22a in 1252ms, sequenceid=479, compaction requested=true at 1731875111290 (+6 ms)Writing region close event to WAL at 1731875111376 (+86 ms)Running coprocessor post-close hooks at 1731875111379 (+3 ms)Closed at 1731875111379 2024-11-17T20:25:11,381 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:11,381 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=31e1f74c01abc6e3eb193ea5bbd4a22a, regionState=CLOSED 2024-11-17T20:25:11,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:25:11,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-17T20:25:11,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 31e1f74c01abc6e3eb193ea5bbd4a22a, server=0b0b0ae8de03,46555,1731875029029 in 1.5320 sec 2024-11-17T20:25:11,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-17T20:25:11,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=31e1f74c01abc6e3eb193ea5bbd4a22a, UNASSIGN in 1.5370 sec 2024-11-17T20:25:11,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-17T20:25:11,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5410 sec 2024-11-17T20:25:11,389 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875111389"}]},"ts":"1731875111389"} 2024-11-17T20:25:11,390 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T20:25:11,390 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T20:25:11,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5550 sec 2024-11-17T20:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T20:25:11,989 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:11,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T20:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:11,991 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=100, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-17T20:25:11,992 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=100, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:11,995 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:11,997 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/recovered.edits] 2024-11-17T20:25:11,999 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/91ba1c2d962d46e7a01384dd9c03ca64 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/91ba1c2d962d46e7a01384dd9c03ca64 2024-11-17T20:25:12,000 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/cacc3b66c1bd4623a276a70fec5473f2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/cacc3b66c1bd4623a276a70fec5473f2 2024-11-17T20:25:12,001 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d8eedc737be34976b8e81e11c65f5128 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/A/d8eedc737be34976b8e81e11c65f5128 2024-11-17T20:25:12,003 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/3e907de5b7f54c5da5f1102a6aa2703c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/3e907de5b7f54c5da5f1102a6aa2703c 2024-11-17T20:25:12,003 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7b02865d9e694a2ca649c3e4c956c380 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/7b02865d9e694a2ca649c3e4c956c380 2024-11-17T20:25:12,004 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ecae93dff58a479d9de5d8852a838961 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/B/ecae93dff58a479d9de5d8852a838961 2024-11-17T20:25:12,006 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/56d74618ddea4f3692c826fc3bb96958 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/56d74618ddea4f3692c826fc3bb96958 2024-11-17T20:25:12,006 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/ee049392f4f14180b2cafc7a8f58aa1d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/ee049392f4f14180b2cafc7a8f58aa1d 2024-11-17T20:25:12,007 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/f6269f03bab9408189862972210d9976 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/C/f6269f03bab9408189862972210d9976 2024-11-17T20:25:12,010 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/recovered.edits/482.seqid to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a/recovered.edits/482.seqid 2024-11-17T20:25:12,010 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/31e1f74c01abc6e3eb193ea5bbd4a22a 2024-11-17T20:25:12,010 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T20:25:12,012 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=100, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:12,016 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T20:25:12,018 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T20:25:12,019 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=100, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:12,019 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T20:25:12,019 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731875112019"}]},"ts":"9223372036854775807"} 2024-11-17T20:25:12,021 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T20:25:12,021 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 31e1f74c01abc6e3eb193ea5bbd4a22a, NAME => 'TestAcidGuarantees,,1731875085891.31e1f74c01abc6e3eb193ea5bbd4a22a.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T20:25:12,021 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T20:25:12,021 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731875112021"}]},"ts":"9223372036854775807"} 2024-11-17T20:25:12,023 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T20:25:12,024 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=100, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:12,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-17T20:25:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-17T20:25:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T20:25:12,103 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:12,108 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 245), OpenFileDescriptor=455 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 499) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8243 (was 8101) - AvailableMemoryMB LEAK? - 2024-11-17T20:25:12,115 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=510, ProcessCount=11, AvailableMemoryMB=8243 2024-11-17T20:25:12,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:25:12,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:12,118 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T20:25:12,118 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:12,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 101 2024-11-17T20:25:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T20:25:12,119 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T20:25:12,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742171_1347 (size=959) 2024-11-17T20:25:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T20:25:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T20:25:12,527 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:25:12,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742172_1348 (size=53) 2024-11-17T20:25:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T20:25:12,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:25:12,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2d8bc42f47e0ab2de330b6f0f05c9c6f, disabling compactions & flushes 2024-11-17T20:25:12,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:12,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:12,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. after waiting 0 ms 2024-11-17T20:25:12,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:12,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:12,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: Waiting for close lock at 1731875112932Disabling compacts and flushes for region at 1731875112932Disabling writes for close at 1731875112932Writing region close event to WAL at 1731875112932Closed at 1731875112932 2024-11-17T20:25:12,934 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T20:25:12,934 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731875112934"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875112934"}]},"ts":"1731875112934"} 2024-11-17T20:25:12,936 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T20:25:12,937 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T20:25:12,937 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875112937"}]},"ts":"1731875112937"} 2024-11-17T20:25:12,939 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T20:25:12,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, ASSIGN}] 2024-11-17T20:25:12,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, ASSIGN 2024-11-17T20:25:12,942 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=false 2024-11-17T20:25:13,093 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:13,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, ASSIGN because future has completed 2024-11-17T20:25:13,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:25:13,256 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:13,256 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:25:13,256 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,256 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:25:13,256 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7794): checking encryption for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,256 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7797): checking classloading for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,258 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,259 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8bc42f47e0ab2de330b6f0f05c9c6f columnFamilyName A 2024-11-17T20:25:13,259 DEBUG [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T20:25:13,260 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(327): Store=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:13,260 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,261 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8bc42f47e0ab2de330b6f0f05c9c6f columnFamilyName B 2024-11-17T20:25:13,261 DEBUG [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:13,262 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(327): Store=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:13,262 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,263 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8bc42f47e0ab2de330b6f0f05c9c6f columnFamilyName C 2024-11-17T20:25:13,263 DEBUG [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:13,263 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(327): Store=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:13,263 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1038): replaying wal for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,264 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,264 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,265 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1048): stopping wal replay for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,265 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1060): Cleaning up temporary data for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,265 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:25:13,266 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): writing seq id for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,268 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:25:13,268 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1114): Opened 2d8bc42f47e0ab2de330b6f0f05c9c6f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67601654, jitterRate=0.0073431432247161865}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:25:13,268 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:13,269 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1006): Region open journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: Running coprocessor pre-open hook at 1731875113257Writing region info on filesystem at 1731875113257Initializing all the Stores at 1731875113257Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875113257Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875113258 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875113258Cleaning up temporary data from old regions at 1731875113265 (+7 ms)Running coprocessor post-open hooks at 1731875113268 (+3 ms)Region opened successfully at 1731875113269 (+1 ms) 2024-11-17T20:25:13,270 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., pid=103, masterSystemTime=1731875113253 2024-11-17T20:25:13,271 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:13,271 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:13,272 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:13,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:25:13,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-17T20:25:13,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 in 176 msec 2024-11-17T20:25:13,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T20:25:13,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, ASSIGN in 336 msec 2024-11-17T20:25:13,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T20:25:13,278 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875113278"}]},"ts":"1731875113278"} 2024-11-17T20:25:13,279 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T20:25:13,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T20:25:13,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-17T20:25:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T20:25:14,271 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:14,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:14,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:14,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:14,276 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c8dae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:14,279 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5015aeb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:14,279 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:14,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:14,282 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T20:25:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,32801,1731875028288 2024-11-17T20:25:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18bbaeb2 2024-11-17T20:25:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T20:25:14,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T20:25:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:25:14,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:25:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:14,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742173_1349 (size=995) 2024-11-17T20:25:14,701 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T20:25:14,701 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T20:25:14,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:25:14,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, REOPEN/MOVE}] 2024-11-17T20:25:14,712 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, REOPEN/MOVE 2024-11-17T20:25:14,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:14,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, REOPEN/MOVE because future has completed 2024-11-17T20:25:14,714 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:25:14,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:25:14,867 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(122): Close 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:14,867 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:25:14,867 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1722): Closing 2d8bc42f47e0ab2de330b6f0f05c9c6f, disabling compactions & flushes 2024-11-17T20:25:14,867 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:14,867 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:14,868 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. after waiting 0 ms 2024-11-17T20:25:14,868 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:14,875 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T20:25:14,876 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:14,876 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1676): Region close journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: Waiting for close lock at 1731875114867Running coprocessor pre-close hooks at 1731875114867Disabling compacts and flushes for region at 1731875114867Disabling writes for close at 1731875114868 (+1 ms)Writing region close event to WAL at 1731875114869 (+1 ms)Running coprocessor post-close hooks at 1731875114875 (+6 ms)Closed at 1731875114876 (+1 ms) 2024-11-17T20:25:14,876 WARN [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegionServer(3299): Not adding moved region record: 2d8bc42f47e0ab2de330b6f0f05c9c6f to self. 2024-11-17T20:25:14,878 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(157): Closed 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:14,879 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=CLOSED 2024-11-17T20:25:14,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:25:14,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-17T20:25:14,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 in 167 msec 2024-11-17T20:25:14,884 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, REOPEN/MOVE; state=CLOSED, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=true 2024-11-17T20:25:15,035 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, REOPEN/MOVE because future has completed 2024-11-17T20:25:15,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=106, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:25:15,199 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,200 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:25:15,201 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,201 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:25:15,201 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,201 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,204 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,206 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8bc42f47e0ab2de330b6f0f05c9c6f columnFamilyName A 2024-11-17T20:25:15,207 DEBUG [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:15,208 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(327): Store=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:15,209 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,210 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8bc42f47e0ab2de330b6f0f05c9c6f columnFamilyName B 2024-11-17T20:25:15,210 DEBUG [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:15,210 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(327): Store=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:15,211 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,211 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8bc42f47e0ab2de330b6f0f05c9c6f columnFamilyName C 2024-11-17T20:25:15,211 DEBUG [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:15,212 INFO [StoreOpener-2d8bc42f47e0ab2de330b6f0f05c9c6f-1 {}] regionserver.HStore(327): Store=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:15,212 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,213 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,214 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,215 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,215 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,215 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:25:15,217 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,218 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 2d8bc42f47e0ab2de330b6f0f05c9c6f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63945180, jitterRate=-0.04714256525039673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:25:15,218 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,219 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: Running coprocessor pre-open hook at 1731875115202Writing region info on filesystem at 1731875115202Initializing all the Stores at 1731875115203 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875115203Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875115204 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875115204Cleaning up temporary data from old regions at 1731875115215 (+11 ms)Running coprocessor post-open hooks at 1731875115218 (+3 ms)Region opened successfully at 1731875115219 (+1 ms) 2024-11-17T20:25:15,220 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., pid=108, masterSystemTime=1731875115194 2024-11-17T20:25:15,222 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,222 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,223 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=OPEN, openSeqNum=5, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=106, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:25:15,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=106 2024-11-17T20:25:15,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=106, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 in 185 msec 2024-11-17T20:25:15,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T20:25:15,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, REOPEN/MOVE in 517 msec 2024-11-17T20:25:15,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-17T20:25:15,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 523 msec 2024-11-17T20:25:15,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-17T20:25:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-17T20:25:15,236 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:15,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,238 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb97db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,240 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c520be8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,242 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341a26e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,244 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f97968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,247 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd842a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,249 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4616603f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,251 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17cbcc5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,253 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2faedc93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c99bf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e7d2b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,260 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b02b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,261 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38077fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f11c15c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,266 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbafc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,268 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7ad2fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,269 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f65477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,271 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51031118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,272 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c20759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:15,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:15,274 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:15,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:15,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:15,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c20054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:15,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a50a8fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:15,276 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,277 DEBUG [Thread-1595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,277 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,277 DEBUG [Thread-1593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:15,277 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,278 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,278 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,278 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T20:25:15,279 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,279 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:15,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,280 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T20:25:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,281 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:15,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:25:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:15,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:15,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111706445f785918493d8384d5e0ae746c4b_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875115290/Put/seqid=0 2024-11-17T20:25:15,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742174_1350 (size=14594) 2024-11-17T20:25:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875175339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875175339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875175344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875175345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875175346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T20:25:15,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875175449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875175449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875175459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875175459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875175459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T20:25:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875175659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875175659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875175669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875175669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875175669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,741 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:15,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,744 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111706445f785918493d8384d5e0ae746c4b_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111706445f785918493d8384d5e0ae746c4b_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:15,745 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b684e9319701447cab0b04a91d202eb5, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:15,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b684e9319701447cab0b04a91d202eb5 is 175, key is test_row_0/A:col10/1731875115290/Put/seqid=0 2024-11-17T20:25:15,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742175_1351 (size=39549) 2024-11-17T20:25:15,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:15,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T20:25:15,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875175969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875175979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875175979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875175980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875175989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:16,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,149 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b684e9319701447cab0b04a91d202eb5 2024-11-17T20:25:16,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/c2ddd2f130fb4cfb9ec7dd78388fb42b is 50, key is test_row_0/B:col10/1731875115290/Put/seqid=0 2024-11-17T20:25:16,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742176_1352 (size=12001) 2024-11-17T20:25:16,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/c2ddd2f130fb4cfb9ec7dd78388fb42b 2024-11-17T20:25:16,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/e21a0454d0d14a559ec78fd5b03390a2 is 50, key is test_row_0/C:col10/1731875115290/Put/seqid=0 2024-11-17T20:25:16,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742177_1353 (size=12001) 2024-11-17T20:25:16,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:16,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:16,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T20:25:16,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875176490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875176499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875176499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875176499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:16,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:16,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875176509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:16,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/e21a0454d0d14a559ec78fd5b03390a2 2024-11-17T20:25:16,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b684e9319701447cab0b04a91d202eb5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b684e9319701447cab0b04a91d202eb5 2024-11-17T20:25:16,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b684e9319701447cab0b04a91d202eb5, entries=200, sequenceid=17, filesize=38.6 K 2024-11-17T20:25:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/c2ddd2f130fb4cfb9ec7dd78388fb42b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/c2ddd2f130fb4cfb9ec7dd78388fb42b 2024-11-17T20:25:16,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/c2ddd2f130fb4cfb9ec7dd78388fb42b, entries=150, sequenceid=17, filesize=11.7 K 2024-11-17T20:25:16,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/e21a0454d0d14a559ec78fd5b03390a2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e21a0454d0d14a559ec78fd5b03390a2 2024-11-17T20:25:16,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e21a0454d0d14a559ec78fd5b03390a2, entries=150, sequenceid=17, filesize=11.7 K 2024-11-17T20:25:16,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1330ms, sequenceid=17, compaction requested=false 2024-11-17T20:25:16,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:16,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T20:25:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:16,663 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T20:25:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174393c45a9f274a33a81ffc7ad080b37e_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875115344/Put/seqid=0 2024-11-17T20:25:16,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742178_1354 (size=12154) 2024-11-17T20:25:16,913 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T20:25:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:17,093 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174393c45a9f274a33a81ffc7ad080b37e_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174393c45a9f274a33a81ffc7ad080b37e_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:17,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/81556b13caac4f05888aadf398eb2037, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:17,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/81556b13caac4f05888aadf398eb2037 is 175, key is test_row_0/A:col10/1731875115344/Put/seqid=0 2024-11-17T20:25:17,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742179_1355 (size=30955) 2024-11-17T20:25:17,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T20:25:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T20:25:17,500 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/81556b13caac4f05888aadf398eb2037 2024-11-17T20:25:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:17,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:17,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/544c2bf1df594acfbf55f672260ddc10 is 50, key is test_row_0/B:col10/1731875115344/Put/seqid=0 2024-11-17T20:25:17,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742180_1356 (size=12001) 2024-11-17T20:25:17,513 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/544c2bf1df594acfbf55f672260ddc10 2024-11-17T20:25:17,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/11a5187f3f6f40418a1f0dbee395764b is 50, key is test_row_0/C:col10/1731875115344/Put/seqid=0 2024-11-17T20:25:17,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742181_1357 (size=12001) 2024-11-17T20:25:17,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875177524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875177525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875177526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875177529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875177529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,536 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875177639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875177639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875177639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875177639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875177850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875177850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875177850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875177850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:17,926 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/11a5187f3f6f40418a1f0dbee395764b 2024-11-17T20:25:17,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/81556b13caac4f05888aadf398eb2037 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/81556b13caac4f05888aadf398eb2037 2024-11-17T20:25:17,935 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/81556b13caac4f05888aadf398eb2037, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T20:25:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/544c2bf1df594acfbf55f672260ddc10 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/544c2bf1df594acfbf55f672260ddc10 2024-11-17T20:25:17,939 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/544c2bf1df594acfbf55f672260ddc10, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T20:25:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/11a5187f3f6f40418a1f0dbee395764b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/11a5187f3f6f40418a1f0dbee395764b 2024-11-17T20:25:17,943 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/11a5187f3f6f40418a1f0dbee395764b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T20:25:17,944 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1281ms, sequenceid=40, compaction requested=false 2024-11-17T20:25:17,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:17,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:17,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T20:25:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T20:25:17,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T20:25:17,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6640 sec 2024-11-17T20:25:17,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6700 sec 2024-11-17T20:25:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:18,163 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:18,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:25:18,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117737d62d8969a4bf6996cf55c6b2ba21b_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875117525/Put/seqid=0 2024-11-17T20:25:18,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742182_1358 (size=17034) 2024-11-17T20:25:18,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875178199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875178236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875178236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875178236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875178339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875178350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875178350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875178350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875178549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875178559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875178560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875178560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:18,576 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117737d62d8969a4bf6996cf55c6b2ba21b_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117737d62d8969a4bf6996cf55c6b2ba21b_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:18,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/72221797b69d482a8cfa13b4bc9c3f5c, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:18,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/72221797b69d482a8cfa13b4bc9c3f5c is 175, key is test_row_0/A:col10/1731875117525/Put/seqid=0 2024-11-17T20:25:18,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742183_1359 (size=48139) 2024-11-17T20:25:18,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/72221797b69d482a8cfa13b4bc9c3f5c 2024-11-17T20:25:18,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e17d72b738d842d384da8e89ea6598b4 is 50, key is test_row_0/B:col10/1731875117525/Put/seqid=0 2024-11-17T20:25:18,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742184_1360 (size=12001) 2024-11-17T20:25:18,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875178859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875178869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875178880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875178880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e17d72b738d842d384da8e89ea6598b4 2024-11-17T20:25:19,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/0d484eb2a40b48cf9b7c8e509ae72147 is 50, key is test_row_0/C:col10/1731875117525/Put/seqid=0 2024-11-17T20:25:19,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742185_1361 (size=12001) 2024-11-17T20:25:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875179369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:19,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875179379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875179390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875179399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/0d484eb2a40b48cf9b7c8e509ae72147 2024-11-17T20:25:19,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/72221797b69d482a8cfa13b4bc9c3f5c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/72221797b69d482a8cfa13b4bc9c3f5c 2024-11-17T20:25:19,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/72221797b69d482a8cfa13b4bc9c3f5c, entries=250, sequenceid=55, filesize=47.0 K 2024-11-17T20:25:19,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e17d72b738d842d384da8e89ea6598b4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e17d72b738d842d384da8e89ea6598b4 2024-11-17T20:25:19,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e17d72b738d842d384da8e89ea6598b4, entries=150, sequenceid=55, filesize=11.7 K 2024-11-17T20:25:19,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/0d484eb2a40b48cf9b7c8e509ae72147 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0d484eb2a40b48cf9b7c8e509ae72147 2024-11-17T20:25:19,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0d484eb2a40b48cf9b7c8e509ae72147, entries=150, sequenceid=55, filesize=11.7 K 2024-11-17T20:25:19,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1271ms, sequenceid=55, compaction requested=true 2024-11-17T20:25:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:19,435 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:19,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:19,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:19,435 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:19,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:19,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:19,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:19,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:19,436 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:19,436 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/A is initiating minor compaction (all files) 2024-11-17T20:25:19,436 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:19,436 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/A in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,436 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/B is initiating minor compaction (all files) 2024-11-17T20:25:19,436 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b684e9319701447cab0b04a91d202eb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/81556b13caac4f05888aadf398eb2037, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/72221797b69d482a8cfa13b4bc9c3f5c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=115.9 K 2024-11-17T20:25:19,436 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/B in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,436 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,436 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/c2ddd2f130fb4cfb9ec7dd78388fb42b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/544c2bf1df594acfbf55f672260ddc10, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e17d72b738d842d384da8e89ea6598b4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=35.2 K 2024-11-17T20:25:19,436 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b684e9319701447cab0b04a91d202eb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/81556b13caac4f05888aadf398eb2037, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/72221797b69d482a8cfa13b4bc9c3f5c] 2024-11-17T20:25:19,436 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ddd2f130fb4cfb9ec7dd78388fb42b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731875115290 2024-11-17T20:25:19,436 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b684e9319701447cab0b04a91d202eb5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731875115290 2024-11-17T20:25:19,437 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 544c2bf1df594acfbf55f672260ddc10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731875115338 2024-11-17T20:25:19,437 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81556b13caac4f05888aadf398eb2037, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731875115338 2024-11-17T20:25:19,437 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e17d72b738d842d384da8e89ea6598b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731875117514 2024-11-17T20:25:19,437 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72221797b69d482a8cfa13b4bc9c3f5c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731875117514 2024-11-17T20:25:19,443 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:19,445 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:19,445 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/0c16fee46d7044db93b86da37426b7ae is 50, key is test_row_0/B:col10/1731875117525/Put/seqid=0 2024-11-17T20:25:19,447 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411174de8575d56de46d2983553f86d8a2dbe_2d8bc42f47e0ab2de330b6f0f05c9c6f store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T20:25:19,449 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411174de8575d56de46d2983553f86d8a2dbe_2d8bc42f47e0ab2de330b6f0f05c9c6f, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:19,449 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174de8575d56de46d2983553f86d8a2dbe_2d8bc42f47e0ab2de330b6f0f05c9c6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:19,449 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:19,450 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-17T20:25:19,451 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T20:25:19,452 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:19,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:19,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742186_1362 (size=12104) 2024-11-17T20:25:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742187_1363 (size=4469) 2024-11-17T20:25:19,463 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/0c16fee46d7044db93b86da37426b7ae as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/0c16fee46d7044db93b86da37426b7ae 2024-11-17T20:25:19,465 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#A#compaction#308 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:19,465 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/eb1e9514bae44b0eb18251da5af7e552 is 175, key is test_row_0/A:col10/1731875117525/Put/seqid=0 2024-11-17T20:25:19,469 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/B of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 0c16fee46d7044db93b86da37426b7ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:19,469 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:19,469 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, priority=13, startTime=1731875119435; duration=0sec 2024-11-17T20:25:19,469 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:19,469 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:B 2024-11-17T20:25:19,470 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:19,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742188_1364 (size=31058) 2024-11-17T20:25:19,470 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:19,470 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/C is initiating minor compaction (all files) 2024-11-17T20:25:19,470 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/C in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,470 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e21a0454d0d14a559ec78fd5b03390a2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/11a5187f3f6f40418a1f0dbee395764b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0d484eb2a40b48cf9b7c8e509ae72147] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=35.2 K 2024-11-17T20:25:19,471 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e21a0454d0d14a559ec78fd5b03390a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731875115290 2024-11-17T20:25:19,471 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a5187f3f6f40418a1f0dbee395764b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731875115338 2024-11-17T20:25:19,471 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d484eb2a40b48cf9b7c8e509ae72147, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731875117514 2024-11-17T20:25:19,477 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:19,478 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/a4cf939a961d4d72b54c226fa2e9784a is 50, key is test_row_0/C:col10/1731875117525/Put/seqid=0 2024-11-17T20:25:19,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742189_1365 (size=12104) 2024-11-17T20:25:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T20:25:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:19,561 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:19,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:25:19,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111796ad32d9a329422388cdbe53a2ecb0dc_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875119560/Put/seqid=0 2024-11-17T20:25:19,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742190_1366 (size=14594) 2024-11-17T20:25:19,572 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:19,576 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111796ad32d9a329422388cdbe53a2ecb0dc_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111796ad32d9a329422388cdbe53a2ecb0dc_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:19,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5f848d05a5174fd48bbf40cbf210255d, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:19,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5f848d05a5174fd48bbf40cbf210255d is 175, key is test_row_0/A:col10/1731875119560/Put/seqid=0 2024-11-17T20:25:19,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742191_1367 (size=39549) 2024-11-17T20:25:19,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T20:25:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875179616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875179729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T20:25:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T20:25:19,874 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/eb1e9514bae44b0eb18251da5af7e552 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/eb1e9514bae44b0eb18251da5af7e552 2024-11-17T20:25:19,878 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/A of 2d8bc42f47e0ab2de330b6f0f05c9c6f into eb1e9514bae44b0eb18251da5af7e552(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:19,879 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:19,879 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, priority=13, startTime=1731875119434; duration=0sec 2024-11-17T20:25:19,879 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:19,879 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:A 2024-11-17T20:25:19,884 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/a4cf939a961d4d72b54c226fa2e9784a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/a4cf939a961d4d72b54c226fa2e9784a 2024-11-17T20:25:19,888 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/C of 2d8bc42f47e0ab2de330b6f0f05c9c6f into a4cf939a961d4d72b54c226fa2e9784a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:19,888 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:19,888 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, priority=13, startTime=1731875119435; duration=0sec 2024-11-17T20:25:19,888 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:19,888 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:C 2024-11-17T20:25:19,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T20:25:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:19,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:19,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875179939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:19,981 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5f848d05a5174fd48bbf40cbf210255d 2024-11-17T20:25:19,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/740f82a04d164200b2f4c77093c021a1 is 50, key is test_row_0/B:col10/1731875119560/Put/seqid=0 2024-11-17T20:25:19,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742192_1368 (size=12001) 2024-11-17T20:25:19,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/740f82a04d164200b2f4c77093c021a1 2024-11-17T20:25:20,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/94d2af67a14340d984e74e7485d721c9 is 50, key is test_row_0/C:col10/1731875119560/Put/seqid=0 2024-11-17T20:25:20,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742193_1369 (size=12001) 2024-11-17T20:25:20,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T20:25:20,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:20,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:20,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:20,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T20:25:20,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T20:25:20,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:20,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:20,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:20,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875180249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:20,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T20:25:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:20,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:20,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875180389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:20,394 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:20,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875180400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:20,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875180400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:20,405 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:20,405 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:20,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/94d2af67a14340d984e74e7485d721c9 2024-11-17T20:25:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5f848d05a5174fd48bbf40cbf210255d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5f848d05a5174fd48bbf40cbf210255d 2024-11-17T20:25:20,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5f848d05a5174fd48bbf40cbf210255d, entries=200, sequenceid=80, filesize=38.6 K 2024-11-17T20:25:20,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/740f82a04d164200b2f4c77093c021a1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/740f82a04d164200b2f4c77093c021a1 2024-11-17T20:25:20,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/740f82a04d164200b2f4c77093c021a1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-17T20:25:20,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/94d2af67a14340d984e74e7485d721c9 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/94d2af67a14340d984e74e7485d721c9 2024-11-17T20:25:20,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875180419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:20,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/94d2af67a14340d984e74e7485d721c9, entries=150, sequenceid=80, filesize=11.7 K 2024-11-17T20:25:20,423 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:20,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 863ms, sequenceid=80, compaction requested=false 2024-11-17T20:25:20,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:20,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T20:25:20,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:20,521 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:20,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a90d56fe9a51447baaae9d4cb521d085_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875119611/Put/seqid=0 2024-11-17T20:25:20,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742194_1370 (size=12154) 2024-11-17T20:25:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:20,534 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a90d56fe9a51447baaae9d4cb521d085_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a90d56fe9a51447baaae9d4cb521d085_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:20,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9b09d529b09344c8b66b1a51e053e4ac, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:20,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9b09d529b09344c8b66b1a51e053e4ac is 175, key is test_row_0/A:col10/1731875119611/Put/seqid=0 2024-11-17T20:25:20,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742195_1371 (size=30955) 2024-11-17T20:25:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T20:25:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:20,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:20,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875180869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:20,940 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9b09d529b09344c8b66b1a51e053e4ac 2024-11-17T20:25:20,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/2fbfae8de4b6430c874862e102fb29a8 is 50, key is test_row_0/B:col10/1731875119611/Put/seqid=0 2024-11-17T20:25:20,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742196_1372 (size=12001) 2024-11-17T20:25:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875180979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875181189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:21,350 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/2fbfae8de4b6430c874862e102fb29a8 2024-11-17T20:25:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/1f3209763a9b4979b6fc7672ee744c22 is 50, key is test_row_0/C:col10/1731875119611/Put/seqid=0 2024-11-17T20:25:21,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742197_1373 (size=12001) 2024-11-17T20:25:21,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875181499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T20:25:21,759 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/1f3209763a9b4979b6fc7672ee744c22 2024-11-17T20:25:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9b09d529b09344c8b66b1a51e053e4ac as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9b09d529b09344c8b66b1a51e053e4ac 2024-11-17T20:25:21,766 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9b09d529b09344c8b66b1a51e053e4ac, entries=150, sequenceid=94, filesize=30.2 K 2024-11-17T20:25:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/2fbfae8de4b6430c874862e102fb29a8 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/2fbfae8de4b6430c874862e102fb29a8 2024-11-17T20:25:21,771 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/2fbfae8de4b6430c874862e102fb29a8, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T20:25:21,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/1f3209763a9b4979b6fc7672ee744c22 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/1f3209763a9b4979b6fc7672ee744c22 2024-11-17T20:25:21,774 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/1f3209763a9b4979b6fc7672ee744c22, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T20:25:21,775 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1253ms, sequenceid=94, compaction requested=true 2024-11-17T20:25:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-17T20:25:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-17T20:25:21,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T20:25:21,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-17T20:25:21,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.3280 sec 2024-11-17T20:25:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:22,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:22,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111716baf974834445f08089d76d76c1e900_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875122023/Put/seqid=0 2024-11-17T20:25:22,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742198_1374 (size=14594) 2024-11-17T20:25:22,036 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111716baf974834445f08089d76d76c1e900_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111716baf974834445f08089d76d76c1e900_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:22,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/d2c22745fc2a41babe84022ec293878d, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:22,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/d2c22745fc2a41babe84022ec293878d is 175, key is test_row_0/A:col10/1731875122023/Put/seqid=0 2024-11-17T20:25:22,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742199_1375 (size=39549) 2024-11-17T20:25:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875182075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875182190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875182399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875182399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,407 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875182409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,419 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875182420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,429 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/d2c22745fc2a41babe84022ec293878d 2024-11-17T20:25:22,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875182449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,460 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 2d8bc42f47e0ab2de330b6f0f05c9c6f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:22,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/ca2f04616a434fc2b0a88e9647f47d1d is 50, key is test_row_0/B:col10/1731875122023/Put/seqid=0 2024-11-17T20:25:22,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742200_1376 (size=12001) 2024-11-17T20:25:22,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/ca2f04616a434fc2b0a88e9647f47d1d 2024-11-17T20:25:22,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/92c66a3216d54e1e8043599bd498c919 is 50, key is test_row_0/C:col10/1731875122023/Put/seqid=0 2024-11-17T20:25:22,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742201_1377 (size=12001) 2024-11-17T20:25:22,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875182719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:22,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:22,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/92c66a3216d54e1e8043599bd498c919 2024-11-17T20:25:22,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/d2c22745fc2a41babe84022ec293878d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/d2c22745fc2a41babe84022ec293878d 2024-11-17T20:25:22,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/d2c22745fc2a41babe84022ec293878d, entries=200, sequenceid=119, filesize=38.6 K 2024-11-17T20:25:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/ca2f04616a434fc2b0a88e9647f47d1d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ca2f04616a434fc2b0a88e9647f47d1d 2024-11-17T20:25:22,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ca2f04616a434fc2b0a88e9647f47d1d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T20:25:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/92c66a3216d54e1e8043599bd498c919 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/92c66a3216d54e1e8043599bd498c919 2024-11-17T20:25:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/92c66a3216d54e1e8043599bd498c919, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T20:25:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 897ms, sequenceid=119, compaction requested=true 2024-11-17T20:25:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:22,921 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:22,921 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,922 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:22,922 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:22,922 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/A is initiating minor compaction (all files) 2024-11-17T20:25:22,922 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/B is initiating minor compaction (all files) 2024-11-17T20:25:22,922 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/B in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:22,922 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/A in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,923 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/0c16fee46d7044db93b86da37426b7ae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/740f82a04d164200b2f4c77093c021a1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/2fbfae8de4b6430c874862e102fb29a8, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ca2f04616a434fc2b0a88e9647f47d1d] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=47.0 K 2024-11-17T20:25:22,923 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/eb1e9514bae44b0eb18251da5af7e552, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5f848d05a5174fd48bbf40cbf210255d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9b09d529b09344c8b66b1a51e053e4ac, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/d2c22745fc2a41babe84022ec293878d] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=137.8 K 2024-11-17T20:25:22,923 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:22,923 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/eb1e9514bae44b0eb18251da5af7e552, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5f848d05a5174fd48bbf40cbf210255d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9b09d529b09344c8b66b1a51e053e4ac, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/d2c22745fc2a41babe84022ec293878d] 2024-11-17T20:25:22,923 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c16fee46d7044db93b86da37426b7ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731875117514 2024-11-17T20:25:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,923 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1e9514bae44b0eb18251da5af7e552, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731875117514 2024-11-17T20:25:22,923 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 740f82a04d164200b2f4c77093c021a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731875118205 2024-11-17T20:25:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,923 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f848d05a5174fd48bbf40cbf210255d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731875118205 2024-11-17T20:25:22,923 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fbfae8de4b6430c874862e102fb29a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875119604 2024-11-17T20:25:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,924 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b09d529b09344c8b66b1a51e053e4ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875119604 2024-11-17T20:25:22,924 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2f04616a434fc2b0a88e9647f47d1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731875120808 2024-11-17T20:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,924 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2c22745fc2a41babe84022ec293878d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731875120808 2024-11-17T20:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,930 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,941 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111790db0647624c4a539e8719010d313138_2d8bc42f47e0ab2de330b6f0f05c9c6f store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,942 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,943 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111790db0647624c4a539e8719010d313138_2d8bc42f47e0ab2de330b6f0f05c9c6f, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:22,943 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111790db0647624c4a539e8719010d313138_2d8bc42f47e0ab2de330b6f0f05c9c6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:22,943 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/73960f5e7f994c3da4dbd990dd3b8404 is 50, key is test_row_0/B:col10/1731875122023/Put/seqid=0 2024-11-17T20:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742202_1378 (size=12241) 2024-11-17T20:25:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,954 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/73960f5e7f994c3da4dbd990dd3b8404 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/73960f5e7f994c3da4dbd990dd3b8404 2024-11-17T20:25:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742203_1379 (size=4469) 2024-11-17T20:25:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,960 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/B of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 73960f5e7f994c3da4dbd990dd3b8404(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,960 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:22,960 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, priority=12, startTime=1731875122921; duration=0sec 2024-11-17T20:25:22,961 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:22,961 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:B 2024-11-17T20:25:22,961 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/C is initiating minor compaction (all files) 2024-11-17T20:25:22,962 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/C in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:22,962 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/a4cf939a961d4d72b54c226fa2e9784a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/94d2af67a14340d984e74e7485d721c9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/1f3209763a9b4979b6fc7672ee744c22, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/92c66a3216d54e1e8043599bd498c919] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=47.0 K 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a4cf939a961d4d72b54c226fa2e9784a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731875117514 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,962 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d2af67a14340d984e74e7485d721c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731875118205 2024-11-17T20:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,963 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3209763a9b4979b6fc7672ee744c22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875119604 2024-11-17T20:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,963 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c66a3216d54e1e8043599bd498c919, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731875120808 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,971 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,971 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/f831994850c3463183b0c533ecccddc4 is 50, key is test_row_0/C:col10/1731875122023/Put/seqid=0 2024-11-17T20:25:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742204_1380 (size=12241) 2024-11-17T20:25:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:23,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e5bb3b4a0c14d2489a2762521bc9a48_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875122041/Put/seqid=0 2024-11-17T20:25:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742205_1381 (size=12254) 2024-11-17T20:25:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e5bb3b4a0c14d2489a2762521bc9a48_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e5bb3b4a0c14d2489a2762521bc9a48_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/20ee589a4f3a434faa7df2a70f27b5a5, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/20ee589a4f3a434faa7df2a70f27b5a5 is 175, key is test_row_0/A:col10/1731875122041/Put/seqid=0 2024-11-17T20:25:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742206_1382 (size=31051) 2024-11-17T20:25:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,358 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#A#compaction#319 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,359 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5bd81afe9f22431f8971737e8de4745e is 175, key is test_row_0/A:col10/1731875122023/Put/seqid=0 2024-11-17T20:25:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742207_1383 (size=31195) 2024-11-17T20:25:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,386 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/f831994850c3463183b0c533ecccddc4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/f831994850c3463183b0c533ecccddc4 2024-11-17T20:25:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,393 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/C of 2d8bc42f47e0ab2de330b6f0f05c9c6f into f831994850c3463183b0c533ecccddc4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:23,393 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:23,393 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, priority=12, startTime=1731875122921; duration=0sec 2024-11-17T20:25:23,394 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:23,394 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:C 2024-11-17T20:25:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:23,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875183543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T20:25:23,622 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:23,622 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-17T20:25:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T20:25:23,624 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:23,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:23,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:23,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875183660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T20:25:23,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/20ee589a4f3a434faa7df2a70f27b5a5 2024-11-17T20:25:23,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/79a762673a4a4fa8a5c65d55d7e00314 is 50, key is test_row_0/B:col10/1731875122041/Put/seqid=0 2024-11-17T20:25:23,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:23,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:23,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:23,779 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5bd81afe9f22431f8971737e8de4745e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5bd81afe9f22431f8971737e8de4745e 2024-11-17T20:25:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:23,785 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/A of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 5bd81afe9f22431f8971737e8de4745e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:23,785 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:23,786 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, priority=12, startTime=1731875122921; duration=0sec 2024-11-17T20:25:23,786 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:23,786 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:A 2024-11-17T20:25:23,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742208_1384 (size=9707) 2024-11-17T20:25:23,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875183870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:23,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:23,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:23,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:23,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:23,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:23,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T20:25:24,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:24,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875184179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:24,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/79a762673a4a4fa8a5c65d55d7e00314 2024-11-17T20:25:24,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/595e7b8bb68d49b389273372762dadb0 is 50, key is test_row_0/C:col10/1731875122041/Put/seqid=0 2024-11-17T20:25:24,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742209_1385 (size=9707) 2024-11-17T20:25:24,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T20:25:24,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:24,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:24,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/595e7b8bb68d49b389273372762dadb0 2024-11-17T20:25:24,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/20ee589a4f3a434faa7df2a70f27b5a5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/20ee589a4f3a434faa7df2a70f27b5a5 2024-11-17T20:25:24,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/20ee589a4f3a434faa7df2a70f27b5a5, entries=150, sequenceid=132, filesize=30.3 K 2024-11-17T20:25:24,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/79a762673a4a4fa8a5c65d55d7e00314 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/79a762673a4a4fa8a5c65d55d7e00314 2024-11-17T20:25:24,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/79a762673a4a4fa8a5c65d55d7e00314, entries=100, sequenceid=132, filesize=9.5 K 2024-11-17T20:25:24,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/595e7b8bb68d49b389273372762dadb0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/595e7b8bb68d49b389273372762dadb0 2024-11-17T20:25:24,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/595e7b8bb68d49b389273372762dadb0, entries=100, sequenceid=132, filesize=9.5 K 2024-11-17T20:25:24,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1403ms, sequenceid=132, compaction requested=false 2024-11-17T20:25:24,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:24,696 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:24,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:24,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:24,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e42484a67904fe091445ab293b09b9e_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875124694/Put/seqid=0 2024-11-17T20:25:24,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742210_1386 (size=14794) 2024-11-17T20:25:24,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:24,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e42484a67904fe091445ab293b09b9e_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e42484a67904fe091445ab293b09b9e_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:24,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/e142b2db779b46e1abff8915b993f976, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:24,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/e142b2db779b46e1abff8915b993f976 is 175, key is test_row_0/A:col10/1731875124694/Put/seqid=0 2024-11-17T20:25:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T20:25:24,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742211_1387 (size=39749) 2024-11-17T20:25:24,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/e142b2db779b46e1abff8915b993f976 2024-11-17T20:25:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875184794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:24,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/ab673929c4ab4c5e8ddf9e5d0956ab91 is 50, key is test_row_0/B:col10/1731875124694/Put/seqid=0 2024-11-17T20:25:24,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742212_1388 (size=12151) 2024-11-17T20:25:24,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/ab673929c4ab4c5e8ddf9e5d0956ab91 2024-11-17T20:25:24,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/3de057c8816c4528961837d4a1f2028d is 50, key is test_row_0/C:col10/1731875124694/Put/seqid=0 2024-11-17T20:25:24,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:24,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:24,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:24,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:24,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742213_1389 (size=12151) 2024-11-17T20:25:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875184910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:25,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875185131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:25,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/3de057c8816c4528961837d4a1f2028d 2024-11-17T20:25:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/e142b2db779b46e1abff8915b993f976 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e142b2db779b46e1abff8915b993f976 2024-11-17T20:25:25,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e142b2db779b46e1abff8915b993f976, entries=200, sequenceid=159, filesize=38.8 K 2024-11-17T20:25:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/ab673929c4ab4c5e8ddf9e5d0956ab91 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ab673929c4ab4c5e8ddf9e5d0956ab91 2024-11-17T20:25:25,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ab673929c4ab4c5e8ddf9e5d0956ab91, entries=150, sequenceid=159, filesize=11.9 K 2024-11-17T20:25:25,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/3de057c8816c4528961837d4a1f2028d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/3de057c8816c4528961837d4a1f2028d 2024-11-17T20:25:25,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/3de057c8816c4528961837d4a1f2028d, entries=150, sequenceid=159, filesize=11.9 K 2024-11-17T20:25:25,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 625ms, sequenceid=159, compaction requested=true 2024-11-17T20:25:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:25,321 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:25,321 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:25,322 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:25,322 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101995 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:25,322 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/A is initiating minor compaction (all files) 2024-11-17T20:25:25,322 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/B is initiating minor compaction (all files) 2024-11-17T20:25:25,322 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/A in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,322 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/B in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,322 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5bd81afe9f22431f8971737e8de4745e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/20ee589a4f3a434faa7df2a70f27b5a5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e142b2db779b46e1abff8915b993f976] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=99.6 K 2024-11-17T20:25:25,322 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/73960f5e7f994c3da4dbd990dd3b8404, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/79a762673a4a4fa8a5c65d55d7e00314, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ab673929c4ab4c5e8ddf9e5d0956ab91] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=33.3 K 2024-11-17T20:25:25,322 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,322 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5bd81afe9f22431f8971737e8de4745e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/20ee589a4f3a434faa7df2a70f27b5a5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e142b2db779b46e1abff8915b993f976] 2024-11-17T20:25:25,322 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 73960f5e7f994c3da4dbd990dd3b8404, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731875120808 2024-11-17T20:25:25,322 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd81afe9f22431f8971737e8de4745e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731875120808 2024-11-17T20:25:25,323 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ee589a4f3a434faa7df2a70f27b5a5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875122041 2024-11-17T20:25:25,323 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a762673a4a4fa8a5c65d55d7e00314, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875122041 2024-11-17T20:25:25,323 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e142b2db779b46e1abff8915b993f976, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875123497 2024-11-17T20:25:25,323 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ab673929c4ab4c5e8ddf9e5d0956ab91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875123530 2024-11-17T20:25:25,341 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:25,342 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#B#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:25,343 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/b79e311edc984355952d56abab5ebb32 is 50, key is test_row_0/B:col10/1731875124694/Put/seqid=0 2024-11-17T20:25:25,345 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111781e56873fedd48b989ce6f84cab6751a_2d8bc42f47e0ab2de330b6f0f05c9c6f store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:25,347 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111781e56873fedd48b989ce6f84cab6751a_2d8bc42f47e0ab2de330b6f0f05c9c6f, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:25,348 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111781e56873fedd48b989ce6f84cab6751a_2d8bc42f47e0ab2de330b6f0f05c9c6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:25,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742214_1390 (size=12493) 2024-11-17T20:25:25,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742215_1391 (size=4469) 2024-11-17T20:25:25,376 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#A#compaction#329 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:25,376 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9d9f7397f9fe477db3c1d6a5c8cc689a is 175, key is test_row_0/A:col10/1731875124694/Put/seqid=0 2024-11-17T20:25:25,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742216_1392 (size=31447) 2024-11-17T20:25:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:25,447 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:25,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:25,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ac98d019cfa549bebe71476c6e1bc5a4_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875124772/Put/seqid=0 2024-11-17T20:25:25,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:25,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:25,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,471 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32801 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=0b0b0ae8de03,46555,1731875029029, table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T20:25:25,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742217_1393 (size=14794) 2024-11-17T20:25:25,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875185638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:25,767 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/b79e311edc984355952d56abab5ebb32 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/b79e311edc984355952d56abab5ebb32 2024-11-17T20:25:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875185759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T20:25:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:25,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:25,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,780 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/B of 2d8bc42f47e0ab2de330b6f0f05c9c6f into b79e311edc984355952d56abab5ebb32(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:25,780 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:25,781 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, priority=13, startTime=1731875125321; duration=0sec 2024-11-17T20:25:25,781 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:25,781 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:B 2024-11-17T20:25:25,781 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:25,782 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:25,782 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/C is initiating minor compaction (all files) 2024-11-17T20:25:25,782 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/C in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,783 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/f831994850c3463183b0c533ecccddc4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/595e7b8bb68d49b389273372762dadb0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/3de057c8816c4528961837d4a1f2028d] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=33.3 K 2024-11-17T20:25:25,783 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f831994850c3463183b0c533ecccddc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731875120808 2024-11-17T20:25:25,783 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 595e7b8bb68d49b389273372762dadb0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875122041 2024-11-17T20:25:25,784 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de057c8816c4528961837d4a1f2028d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875123530 2024-11-17T20:25:25,808 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9d9f7397f9fe477db3c1d6a5c8cc689a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9d9f7397f9fe477db3c1d6a5c8cc689a 2024-11-17T20:25:25,813 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#C#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:25,813 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/5d37fc8fbf1e4f51b7e204d99084bf12 is 50, key is test_row_0/C:col10/1731875124694/Put/seqid=0 2024-11-17T20:25:25,814 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/A of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 9d9f7397f9fe477db3c1d6a5c8cc689a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:25,814 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:25,814 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, priority=13, startTime=1731875125321; duration=0sec 2024-11-17T20:25:25,814 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:25,814 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:A 2024-11-17T20:25:25,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742218_1394 (size=12493) 2024-11-17T20:25:25,840 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/5d37fc8fbf1e4f51b7e204d99084bf12 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5d37fc8fbf1e4f51b7e204d99084bf12 2024-11-17T20:25:25,845 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/C of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 5d37fc8fbf1e4f51b7e204d99084bf12(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:25,845 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:25,845 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, priority=13, startTime=1731875125321; duration=0sec 2024-11-17T20:25:25,845 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:25,845 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:C 2024-11-17T20:25:25,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:25,877 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ac98d019cfa549bebe71476c6e1bc5a4_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ac98d019cfa549bebe71476c6e1bc5a4_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:25,877 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ff543dff2d3f459594db7c8bd041687c, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:25,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ff543dff2d3f459594db7c8bd041687c is 175, key is test_row_0/A:col10/1731875124772/Put/seqid=0 2024-11-17T20:25:25,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742219_1395 (size=39749) 2024-11-17T20:25:25,881 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ff543dff2d3f459594db7c8bd041687c 2024-11-17T20:25:25,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/87a7d22c649145da9e5bb431a246b0cd is 50, key is test_row_0/B:col10/1731875124772/Put/seqid=0 2024-11-17T20:25:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742220_1396 (size=12151) 2024-11-17T20:25:25,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/87a7d22c649145da9e5bb431a246b0cd 2024-11-17T20:25:25,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/68fc1f0fccb14594ae97d5031de59ae2 is 50, key is test_row_0/C:col10/1731875124772/Put/seqid=0 2024-11-17T20:25:25,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:25,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:25,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742221_1397 (size=12151) 2024-11-17T20:25:25,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/68fc1f0fccb14594ae97d5031de59ae2 2024-11-17T20:25:25,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ff543dff2d3f459594db7c8bd041687c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ff543dff2d3f459594db7c8bd041687c 2024-11-17T20:25:25,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ff543dff2d3f459594db7c8bd041687c, entries=200, sequenceid=171, filesize=38.8 K 2024-11-17T20:25:25,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/87a7d22c649145da9e5bb431a246b0cd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/87a7d22c649145da9e5bb431a246b0cd 2024-11-17T20:25:25,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/87a7d22c649145da9e5bb431a246b0cd, entries=150, sequenceid=171, filesize=11.9 K 2024-11-17T20:25:25,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/68fc1f0fccb14594ae97d5031de59ae2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/68fc1f0fccb14594ae97d5031de59ae2 2024-11-17T20:25:25,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/68fc1f0fccb14594ae97d5031de59ae2, entries=150, sequenceid=171, filesize=11.9 K 2024-11-17T20:25:25,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 513ms, sequenceid=171, compaction requested=false 2024-11-17T20:25:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:25,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:25,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:26,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117528b40d64a5e47ca961ca1fadf741cb4_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875125591/Put/seqid=0 2024-11-17T20:25:26,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742222_1398 (size=14794) 2024-11-17T20:25:26,057 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:26,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117528b40d64a5e47ca961ca1fadf741cb4_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117528b40d64a5e47ca961ca1fadf741cb4_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:26,063 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/f04f4c159c4c455a984e1ed4f0cade22, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:26,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/f04f4c159c4c455a984e1ed4f0cade22 is 175, key is test_row_0/A:col10/1731875125591/Put/seqid=0 2024-11-17T20:25:26,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875186077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742223_1399 (size=39749) 2024-11-17T20:25:26,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/f04f4c159c4c455a984e1ed4f0cade22 2024-11-17T20:25:26,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:26,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:26,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:26,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:26,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/fed208ef68804d799d93a30d57eed024 is 50, key is test_row_0/B:col10/1731875125591/Put/seqid=0 2024-11-17T20:25:26,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742224_1400 (size=12151) 2024-11-17T20:25:26,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/fed208ef68804d799d93a30d57eed024 2024-11-17T20:25:26,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/06be9b6225fc4d6791a2e84489684d4e is 50, key is test_row_0/C:col10/1731875125591/Put/seqid=0 2024-11-17T20:25:26,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742225_1401 (size=12151) 2024-11-17T20:25:26,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/06be9b6225fc4d6791a2e84489684d4e 2024-11-17T20:25:26,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/f04f4c159c4c455a984e1ed4f0cade22 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/f04f4c159c4c455a984e1ed4f0cade22 2024-11-17T20:25:26,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/f04f4c159c4c455a984e1ed4f0cade22, entries=200, sequenceid=199, filesize=38.8 K 2024-11-17T20:25:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/fed208ef68804d799d93a30d57eed024 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/fed208ef68804d799d93a30d57eed024 2024-11-17T20:25:26,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/fed208ef68804d799d93a30d57eed024, entries=150, sequenceid=199, filesize=11.9 K 2024-11-17T20:25:26,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/06be9b6225fc4d6791a2e84489684d4e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/06be9b6225fc4d6791a2e84489684d4e 2024-11-17T20:25:26,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875186191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/06be9b6225fc4d6791a2e84489684d4e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-17T20:25:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 208ms, sequenceid=199, compaction requested=true 2024-11-17T20:25:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:26,201 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:26,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T20:25:26,201 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:26,202 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:26,202 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/B is initiating minor compaction (all files) 2024-11-17T20:25:26,202 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/B in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,202 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/b79e311edc984355952d56abab5ebb32, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/87a7d22c649145da9e5bb431a246b0cd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/fed208ef68804d799d93a30d57eed024] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=35.9 K 2024-11-17T20:25:26,202 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:26,202 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/A is initiating minor compaction (all files) 2024-11-17T20:25:26,203 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/A in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,203 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9d9f7397f9fe477db3c1d6a5c8cc689a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ff543dff2d3f459594db7c8bd041687c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/f04f4c159c4c455a984e1ed4f0cade22] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=108.3 K 2024-11-17T20:25:26,203 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,203 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9d9f7397f9fe477db3c1d6a5c8cc689a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ff543dff2d3f459594db7c8bd041687c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/f04f4c159c4c455a984e1ed4f0cade22] 2024-11-17T20:25:26,203 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting b79e311edc984355952d56abab5ebb32, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875123530 2024-11-17T20:25:26,203 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9f7397f9fe477db3c1d6a5c8cc689a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875123530 2024-11-17T20:25:26,203 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a7d22c649145da9e5bb431a246b0cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731875124772 2024-11-17T20:25:26,203 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff543dff2d3f459594db7c8bd041687c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731875124772 2024-11-17T20:25:26,203 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting fed208ef68804d799d93a30d57eed024, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731875125591 2024-11-17T20:25:26,203 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f04f4c159c4c455a984e1ed4f0cade22, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731875125591 2024-11-17T20:25:26,213 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:26,216 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:26,217 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/a53403b906da4edf89b44fd1c7d59556 is 50, key is test_row_0/B:col10/1731875125591/Put/seqid=0 2024-11-17T20:25:26,219 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117b298995e562c4a2f99974fdb04f333d6_2d8bc42f47e0ab2de330b6f0f05c9c6f store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:26,223 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117b298995e562c4a2f99974fdb04f333d6_2d8bc42f47e0ab2de330b6f0f05c9c6f, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:26,223 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b298995e562c4a2f99974fdb04f333d6_2d8bc42f47e0ab2de330b6f0f05c9c6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:26,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T20:25:26,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742226_1402 (size=12595) 2024-11-17T20:25:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,249 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:25:26,270 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/a53403b906da4edf89b44fd1c7d59556 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/a53403b906da4edf89b44fd1c7d59556 2024-11-17T20:25:26,277 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/B of 2d8bc42f47e0ab2de330b6f0f05c9c6f into a53403b906da4edf89b44fd1c7d59556(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:26,277 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:26,277 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, priority=13, startTime=1731875126201; duration=0sec 2024-11-17T20:25:26,278 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:26,278 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:B 2024-11-17T20:25:26,278 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:26,279 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:26,279 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/C is initiating minor compaction (all files) 2024-11-17T20:25:26,279 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/C in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,279 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5d37fc8fbf1e4f51b7e204d99084bf12, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/68fc1f0fccb14594ae97d5031de59ae2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/06be9b6225fc4d6791a2e84489684d4e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=35.9 K 2024-11-17T20:25:26,282 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d37fc8fbf1e4f51b7e204d99084bf12, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875123530 2024-11-17T20:25:26,283 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 68fc1f0fccb14594ae97d5031de59ae2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731875124772 2024-11-17T20:25:26,283 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 06be9b6225fc4d6791a2e84489684d4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731875125591 2024-11-17T20:25:26,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742227_1403 (size=4469) 2024-11-17T20:25:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bf9b98aa5a5241ba93a628915e12b6fd_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875126075/Put/seqid=0 2024-11-17T20:25:26,309 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#C#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:26,310 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/6b51508091f343be960a646adf60b9b5 is 50, key is test_row_0/C:col10/1731875125591/Put/seqid=0 2024-11-17T20:25:26,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742228_1404 (size=12304) 2024-11-17T20:25:26,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:26,372 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bf9b98aa5a5241ba93a628915e12b6fd_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bf9b98aa5a5241ba93a628915e12b6fd_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:26,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c38ec9f0fefa47a6896c0817d7409e6b, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:26,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c38ec9f0fefa47a6896c0817d7409e6b is 175, key is test_row_0/A:col10/1731875126075/Put/seqid=0 2024-11-17T20:25:26,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742229_1405 (size=12595) 2024-11-17T20:25:26,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742230_1406 (size=31105) 2024-11-17T20:25:26,394 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c38ec9f0fefa47a6896c0817d7409e6b 2024-11-17T20:25:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:26,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:26,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/5ef66b894a864cc38007069e817681ff is 50, key is test_row_0/B:col10/1731875126075/Put/seqid=0 2024-11-17T20:25:26,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742231_1407 (size=12151) 2024-11-17T20:25:26,439 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/5ef66b894a864cc38007069e817681ff 2024-11-17T20:25:26,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/52193e67a17e4a3090a597d2eb743e7e is 50, key is test_row_0/C:col10/1731875126075/Put/seqid=0 2024-11-17T20:25:26,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742232_1408 (size=12151) 2024-11-17T20:25:26,454 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/52193e67a17e4a3090a597d2eb743e7e 2024-11-17T20:25:26,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c38ec9f0fefa47a6896c0817d7409e6b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c38ec9f0fefa47a6896c0817d7409e6b 2024-11-17T20:25:26,465 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c38ec9f0fefa47a6896c0817d7409e6b, entries=150, sequenceid=210, filesize=30.4 K 2024-11-17T20:25:26,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/5ef66b894a864cc38007069e817681ff as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/5ef66b894a864cc38007069e817681ff 2024-11-17T20:25:26,470 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/5ef66b894a864cc38007069e817681ff, entries=150, sequenceid=210, filesize=11.9 K 2024-11-17T20:25:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/52193e67a17e4a3090a597d2eb743e7e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/52193e67a17e4a3090a597d2eb743e7e 2024-11-17T20:25:26,475 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/52193e67a17e4a3090a597d2eb743e7e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-17T20:25:26,476 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=114.05 KB/116790 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 227ms, sequenceid=210, compaction requested=false 2024-11-17T20:25:26,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:26,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:26,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-17T20:25:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:26,477 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:26,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T20:25:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-17T20:25:26,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T20:25:26,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8530 sec 2024-11-17T20:25:26,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.8590 sec 2024-11-17T20:25:26,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178a6e1bd84cba4cda8e4355171ca58b2a_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875126466/Put/seqid=0 2024-11-17T20:25:26,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742233_1409 (size=17284) 2024-11-17T20:25:26,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875186498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875186499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875186500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875186508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875186509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875186619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875186619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875186620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875186629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875186629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,687 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#A#compaction#337 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:26,688 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/e93dbe0190054506a4b3ec6e9718ada4 is 175, key is test_row_0/A:col10/1731875125591/Put/seqid=0 2024-11-17T20:25:26,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742234_1410 (size=31549) 2024-11-17T20:25:26,694 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/e93dbe0190054506a4b3ec6e9718ada4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e93dbe0190054506a4b3ec6e9718ada4 2024-11-17T20:25:26,698 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/A of 2d8bc42f47e0ab2de330b6f0f05c9c6f into e93dbe0190054506a4b3ec6e9718ada4(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:26,698 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:26,698 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, priority=13, startTime=1731875126201; duration=0sec 2024-11-17T20:25:26,698 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:26,698 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:A 2024-11-17T20:25:26,791 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/6b51508091f343be960a646adf60b9b5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/6b51508091f343be960a646adf60b9b5 2024-11-17T20:25:26,796 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/C of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 6b51508091f343be960a646adf60b9b5(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:26,796 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:26,796 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, priority=13, startTime=1731875126201; duration=0sec 2024-11-17T20:25:26,796 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:26,796 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:C 2024-11-17T20:25:26,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875186839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875186840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875186840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875186840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875186840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:26,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:26,893 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178a6e1bd84cba4cda8e4355171ca58b2a_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178a6e1bd84cba4cda8e4355171ca58b2a_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:26,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c9d5401bd37d43789e360eb6e93cdd7a, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:26,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c9d5401bd37d43789e360eb6e93cdd7a is 175, key is test_row_0/A:col10/1731875126466/Put/seqid=0 2024-11-17T20:25:26,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742235_1411 (size=48389) 2024-11-17T20:25:26,897 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c9d5401bd37d43789e360eb6e93cdd7a 2024-11-17T20:25:26,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/dbd1f37381cc45c085ee350e5fb3095e is 50, key is test_row_0/B:col10/1731875126466/Put/seqid=0 2024-11-17T20:25:26,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742236_1412 (size=12151) 2024-11-17T20:25:27,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875187149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875187149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875187150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875187150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875187159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/dbd1f37381cc45c085ee350e5fb3095e 2024-11-17T20:25:27,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/e4c736cd0ea64ce19ecaa983404d5764 is 50, key is test_row_0/C:col10/1731875126466/Put/seqid=0 2024-11-17T20:25:27,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742237_1413 (size=12151) 2024-11-17T20:25:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875187659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875187660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875187670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875187670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875187671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:27,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/e4c736cd0ea64ce19ecaa983404d5764 2024-11-17T20:25:27,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/c9d5401bd37d43789e360eb6e93cdd7a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c9d5401bd37d43789e360eb6e93cdd7a 2024-11-17T20:25:27,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c9d5401bd37d43789e360eb6e93cdd7a, entries=250, sequenceid=233, filesize=47.3 K 2024-11-17T20:25:27,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/dbd1f37381cc45c085ee350e5fb3095e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/dbd1f37381cc45c085ee350e5fb3095e 2024-11-17T20:25:27,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/dbd1f37381cc45c085ee350e5fb3095e, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T20:25:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/e4c736cd0ea64ce19ecaa983404d5764 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e4c736cd0ea64ce19ecaa983404d5764 2024-11-17T20:25:27,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e4c736cd0ea64ce19ecaa983404d5764, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T20:25:27,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1265ms, sequenceid=233, compaction requested=true 2024-11-17T20:25:27,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:27,742 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:27,743 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:27,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:27,744 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:27,744 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:27,744 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/B is initiating minor compaction (all files) 2024-11-17T20:25:27,744 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/A is initiating minor compaction (all files) 2024-11-17T20:25:27,744 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/B in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:27,744 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/A in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:27,744 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/a53403b906da4edf89b44fd1c7d59556, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/5ef66b894a864cc38007069e817681ff, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/dbd1f37381cc45c085ee350e5fb3095e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=36.0 K 2024-11-17T20:25:27,744 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e93dbe0190054506a4b3ec6e9718ada4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c38ec9f0fefa47a6896c0817d7409e6b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c9d5401bd37d43789e360eb6e93cdd7a] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=108.4 K 2024-11-17T20:25:27,744 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:27,744 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e93dbe0190054506a4b3ec6e9718ada4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c38ec9f0fefa47a6896c0817d7409e6b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c9d5401bd37d43789e360eb6e93cdd7a] 2024-11-17T20:25:27,744 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a53403b906da4edf89b44fd1c7d59556, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731875125591 2024-11-17T20:25:27,745 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e93dbe0190054506a4b3ec6e9718ada4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731875125591 2024-11-17T20:25:27,745 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef66b894a864cc38007069e817681ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875126062 2024-11-17T20:25:27,745 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38ec9f0fefa47a6896c0817d7409e6b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875126062 2024-11-17T20:25:27,745 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting dbd1f37381cc45c085ee350e5fb3095e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731875126464 2024-11-17T20:25:27,745 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9d5401bd37d43789e360eb6e93cdd7a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731875126463 2024-11-17T20:25:27,753 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:27,753 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/bde65388c49845f1b5ed7afe2a85ca3e is 50, key is test_row_0/B:col10/1731875126466/Put/seqid=0 2024-11-17T20:25:27,765 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:27,773 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111768503043a856447fbb5869f4e837d5d1_2d8bc42f47e0ab2de330b6f0f05c9c6f store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:27,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742238_1414 (size=12697) 2024-11-17T20:25:27,774 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111768503043a856447fbb5869f4e837d5d1_2d8bc42f47e0ab2de330b6f0f05c9c6f, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:27,774 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111768503043a856447fbb5869f4e837d5d1_2d8bc42f47e0ab2de330b6f0f05c9c6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:27,780 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/bde65388c49845f1b5ed7afe2a85ca3e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bde65388c49845f1b5ed7afe2a85ca3e 2024-11-17T20:25:27,783 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/B of 2d8bc42f47e0ab2de330b6f0f05c9c6f into bde65388c49845f1b5ed7afe2a85ca3e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:27,783 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:27,783 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, priority=13, startTime=1731875127743; duration=0sec 2024-11-17T20:25:27,784 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:27,784 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:B 2024-11-17T20:25:27,784 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:27,785 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:27,785 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/C is initiating minor compaction (all files) 2024-11-17T20:25:27,785 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/C in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:27,786 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/6b51508091f343be960a646adf60b9b5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/52193e67a17e4a3090a597d2eb743e7e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e4c736cd0ea64ce19ecaa983404d5764] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=36.0 K 2024-11-17T20:25:27,786 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b51508091f343be960a646adf60b9b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731875125591 2024-11-17T20:25:27,787 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 52193e67a17e4a3090a597d2eb743e7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731875126062 2024-11-17T20:25:27,787 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c736cd0ea64ce19ecaa983404d5764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731875126464 2024-11-17T20:25:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T20:25:27,789 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:27,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742239_1415 (size=4469) 2024-11-17T20:25:27,790 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-17T20:25:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T20:25:27,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:27,792 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:27,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:27,794 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#C#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:27,795 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/fac30a0d63094b79ab45fa8528dfab59 is 50, key is test_row_0/C:col10/1731875126466/Put/seqid=0 2024-11-17T20:25:27,801 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#A#compaction#347 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:27,802 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/71c37bfc855b40eaaa8a94008ead1b1d is 175, key is test_row_0/A:col10/1731875126466/Put/seqid=0 2024-11-17T20:25:27,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742240_1416 (size=12697) 2024-11-17T20:25:27,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742241_1417 (size=31651) 2024-11-17T20:25:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T20:25:27,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-17T20:25:27,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:27,945 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:25:27,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c69f0140f2974a8292a09ccd6e5b32f3_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875126506/Put/seqid=0 2024-11-17T20:25:27,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742242_1418 (size=12304) 2024-11-17T20:25:27,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:27,966 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c69f0140f2974a8292a09ccd6e5b32f3_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c69f0140f2974a8292a09ccd6e5b32f3_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:27,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/2cbaf64cbe49446185ff319fd1fcdaed, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:27,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/2cbaf64cbe49446185ff319fd1fcdaed is 175, key is test_row_0/A:col10/1731875126506/Put/seqid=0 2024-11-17T20:25:27,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742243_1419 (size=31105) 2024-11-17T20:25:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T20:25:28,212 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/fac30a0d63094b79ab45fa8528dfab59 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fac30a0d63094b79ab45fa8528dfab59 2024-11-17T20:25:28,216 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/C of 2d8bc42f47e0ab2de330b6f0f05c9c6f into fac30a0d63094b79ab45fa8528dfab59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:28,216 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:28,216 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, priority=13, startTime=1731875127743; duration=0sec 2024-11-17T20:25:28,216 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:28,216 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:C 2024-11-17T20:25:28,221 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/71c37bfc855b40eaaa8a94008ead1b1d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/71c37bfc855b40eaaa8a94008ead1b1d 2024-11-17T20:25:28,223 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/A of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 71c37bfc855b40eaaa8a94008ead1b1d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:28,224 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:28,224 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, priority=13, startTime=1731875127742; duration=0sec 2024-11-17T20:25:28,224 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:28,224 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:A 2024-11-17T20:25:28,377 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/2cbaf64cbe49446185ff319fd1fcdaed 2024-11-17T20:25:28,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9588eaa2b2ed43c082711050b8ed65e7 is 50, key is test_row_0/B:col10/1731875126506/Put/seqid=0 2024-11-17T20:25:28,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742244_1420 (size=12151) 2024-11-17T20:25:28,386 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9588eaa2b2ed43c082711050b8ed65e7 2024-11-17T20:25:28,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb is 50, key is test_row_0/C:col10/1731875126506/Put/seqid=0 2024-11-17T20:25:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742245_1421 (size=12151) 2024-11-17T20:25:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T20:25:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:28,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875188705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875188708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875188708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875188713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875188713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,805 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb 2024-11-17T20:25:28,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/2cbaf64cbe49446185ff319fd1fcdaed as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2cbaf64cbe49446185ff319fd1fcdaed 2024-11-17T20:25:28,812 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2cbaf64cbe49446185ff319fd1fcdaed, entries=150, sequenceid=250, filesize=30.4 K 2024-11-17T20:25:28,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9588eaa2b2ed43c082711050b8ed65e7 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9588eaa2b2ed43c082711050b8ed65e7 2024-11-17T20:25:28,816 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9588eaa2b2ed43c082711050b8ed65e7, entries=150, sequenceid=250, filesize=11.9 K 2024-11-17T20:25:28,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb 2024-11-17T20:25:28,821 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb, entries=150, sequenceid=250, filesize=11.9 K 2024-11-17T20:25:28,822 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 877ms, sequenceid=250, compaction requested=false 2024-11-17T20:25:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-17T20:25:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-17T20:25:28,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-17T20:25:28,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0310 sec 2024-11-17T20:25:28,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0350 sec 2024-11-17T20:25:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:28,827 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:28,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T20:25:28,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c95a9d2c901d4a75a0a4f3c0d5cb2a52_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875128712/Put/seqid=0 2024-11-17T20:25:28,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742246_1422 (size=14994) 2024-11-17T20:25:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875188839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875188842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875188843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875188843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875188849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T20:25:28,930 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:28,930 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-17T20:25:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T20:25:28,933 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:28,933 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:28,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:28,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875188959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875188959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875188959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875188959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875188959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T20:25:29,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:29,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:29,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875189169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875189170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875189170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875189170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875189170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:29,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:29,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T20:25:29,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c95a9d2c901d4a75a0a4f3c0d5cb2a52_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c95a9d2c901d4a75a0a4f3c0d5cb2a52_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:29,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/362a27477d714b79bcab062f7739c25e, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:29,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/362a27477d714b79bcab062f7739c25e is 175, key is test_row_0/A:col10/1731875128712/Put/seqid=0 2024-11-17T20:25:29,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742247_1423 (size=39949) 2024-11-17T20:25:29,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:29,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:29,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875189479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875189479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875189489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875189490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875189490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T20:25:29,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/362a27477d714b79bcab062f7739c25e 2024-11-17T20:25:29,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/3fb6a66293764c8381a5579794c88ba4 is 50, key is test_row_0/B:col10/1731875128712/Put/seqid=0 2024-11-17T20:25:29,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742248_1424 (size=12301) 2024-11-17T20:25:29,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:29,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:29,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875189990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875189990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:30,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875189999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:30,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875189999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:30,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875189999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:30,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:30,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T20:25:30,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/3fb6a66293764c8381a5579794c88ba4 2024-11-17T20:25:30,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/acd2db9543e24d7faf88f0b46c27bf1b is 50, key is test_row_0/C:col10/1731875128712/Put/seqid=0 2024-11-17T20:25:30,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742249_1425 (size=12301) 2024-11-17T20:25:30,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/acd2db9543e24d7faf88f0b46c27bf1b 2024-11-17T20:25:30,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/362a27477d714b79bcab062f7739c25e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/362a27477d714b79bcab062f7739c25e 2024-11-17T20:25:30,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/362a27477d714b79bcab062f7739c25e, entries=200, sequenceid=275, filesize=39.0 K 2024-11-17T20:25:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/3fb6a66293764c8381a5579794c88ba4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/3fb6a66293764c8381a5579794c88ba4 2024-11-17T20:25:30,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/3fb6a66293764c8381a5579794c88ba4, entries=150, sequenceid=275, filesize=12.0 K 2024-11-17T20:25:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/acd2db9543e24d7faf88f0b46c27bf1b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/acd2db9543e24d7faf88f0b46c27bf1b 2024-11-17T20:25:30,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/acd2db9543e24d7faf88f0b46c27bf1b, entries=150, sequenceid=275, filesize=12.0 K 2024-11-17T20:25:30,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1305ms, sequenceid=275, compaction requested=true 2024-11-17T20:25:30,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:30,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:30,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:30,132 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:30,133 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:30,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:30,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:30,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:30,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:30,133 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:30,133 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:30,133 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/A is initiating minor compaction (all files) 2024-11-17T20:25:30,133 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/B is initiating minor compaction (all files) 2024-11-17T20:25:30,134 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/B in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:30,134 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/A in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:30,134 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/71c37bfc855b40eaaa8a94008ead1b1d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2cbaf64cbe49446185ff319fd1fcdaed, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/362a27477d714b79bcab062f7739c25e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=100.3 K 2024-11-17T20:25:30,134 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bde65388c49845f1b5ed7afe2a85ca3e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9588eaa2b2ed43c082711050b8ed65e7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/3fb6a66293764c8381a5579794c88ba4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=36.3 K 2024-11-17T20:25:30,134 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:30,134 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/71c37bfc855b40eaaa8a94008ead1b1d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2cbaf64cbe49446185ff319fd1fcdaed, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/362a27477d714b79bcab062f7739c25e] 2024-11-17T20:25:30,136 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bde65388c49845f1b5ed7afe2a85ca3e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731875126464 2024-11-17T20:25:30,136 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c37bfc855b40eaaa8a94008ead1b1d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731875126464 2024-11-17T20:25:30,137 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9588eaa2b2ed43c082711050b8ed65e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731875126497 2024-11-17T20:25:30,137 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cbaf64cbe49446185ff319fd1fcdaed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731875126497 2024-11-17T20:25:30,137 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb6a66293764c8381a5579794c88ba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731875128707 2024-11-17T20:25:30,137 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 362a27477d714b79bcab062f7739c25e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731875128707 2024-11-17T20:25:30,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-17T20:25:30,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:30,162 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T20:25:30,173 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:30,182 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:30,183 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/39bab2f8027c406a85d256405ab5f5eb is 50, key is test_row_0/B:col10/1731875128712/Put/seqid=0 2024-11-17T20:25:30,189 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117cf1bf12845ff4ca38c1b380b7de0c0a6_2d8bc42f47e0ab2de330b6f0f05c9c6f store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:30,191 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117cf1bf12845ff4ca38c1b380b7de0c0a6_2d8bc42f47e0ab2de330b6f0f05c9c6f, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:30,191 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cf1bf12845ff4ca38c1b380b7de0c0a6_2d8bc42f47e0ab2de330b6f0f05c9c6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:30,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111710e34a2923d446bdbbdb03252be547f5_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875128840/Put/seqid=0 2024-11-17T20:25:30,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742250_1426 (size=12949) 2024-11-17T20:25:30,232 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/39bab2f8027c406a85d256405ab5f5eb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/39bab2f8027c406a85d256405ab5f5eb 2024-11-17T20:25:30,238 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/B of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 39bab2f8027c406a85d256405ab5f5eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:30,238 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:30,238 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, priority=13, startTime=1731875130132; duration=0sec 2024-11-17T20:25:30,238 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:30,238 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:B 2024-11-17T20:25:30,238 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:30,239 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:30,239 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/C is initiating minor compaction (all files) 2024-11-17T20:25:30,239 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/C in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:30,239 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fac30a0d63094b79ab45fa8528dfab59, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/acd2db9543e24d7faf88f0b46c27bf1b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=36.3 K 2024-11-17T20:25:30,240 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting fac30a0d63094b79ab45fa8528dfab59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731875126464 2024-11-17T20:25:30,240 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a0132a5a4b407ea0b68ee7b1b8a7eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731875126497 2024-11-17T20:25:30,240 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting acd2db9543e24d7faf88f0b46c27bf1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731875128707 2024-11-17T20:25:30,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742251_1427 (size=4469) 2024-11-17T20:25:30,250 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#A#compaction#355 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:30,250 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/107a2f18f9b14554a15cd541fc81da9a is 175, key is test_row_0/A:col10/1731875128712/Put/seqid=0 2024-11-17T20:25:30,259 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#C#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:30,259 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/44771275df3b4966b49e7936655a07a3 is 50, key is test_row_0/C:col10/1731875128712/Put/seqid=0 2024-11-17T20:25:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742252_1428 (size=12454) 2024-11-17T20:25:30,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742253_1429 (size=31903) 2024-11-17T20:25:30,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742254_1430 (size=12949) 2024-11-17T20:25:30,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:30,689 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111710e34a2923d446bdbbdb03252be547f5_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111710e34a2923d446bdbbdb03252be547f5_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ab6206b6245a4c00bdc00616fc8bbcb5, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ab6206b6245a4c00bdc00616fc8bbcb5 is 175, key is test_row_0/A:col10/1731875128840/Put/seqid=0 2024-11-17T20:25:30,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742255_1431 (size=31255) 2024-11-17T20:25:30,696 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ab6206b6245a4c00bdc00616fc8bbcb5 2024-11-17T20:25:30,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e57c8ee2661e4854a1982e27aac1776d is 50, key is test_row_0/B:col10/1731875128840/Put/seqid=0 2024-11-17T20:25:30,725 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/107a2f18f9b14554a15cd541fc81da9a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/107a2f18f9b14554a15cd541fc81da9a 2024-11-17T20:25:30,733 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/44771275df3b4966b49e7936655a07a3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/44771275df3b4966b49e7936655a07a3 2024-11-17T20:25:30,733 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/A of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 107a2f18f9b14554a15cd541fc81da9a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:30,733 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:30,733 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, priority=13, startTime=1731875130132; duration=0sec 2024-11-17T20:25:30,733 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:30,733 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:A 2024-11-17T20:25:30,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742256_1432 (size=12301) 2024-11-17T20:25:30,740 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e57c8ee2661e4854a1982e27aac1776d 2024-11-17T20:25:30,740 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/C of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 44771275df3b4966b49e7936655a07a3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:30,740 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:30,740 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, priority=13, startTime=1731875130133; duration=0sec 2024-11-17T20:25:30,741 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:30,741 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:C 2024-11-17T20:25:30,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/914c500cdf0d4b06939f0fc34bbad56e is 50, key is test_row_0/C:col10/1731875128840/Put/seqid=0 2024-11-17T20:25:30,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742257_1433 (size=12301) 2024-11-17T20:25:31,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:31,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875191043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875191044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875191046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875191047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875191049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T20:25:31,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875191160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875191160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875191161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875191161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875191171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,209 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/914c500cdf0d4b06939f0fc34bbad56e 2024-11-17T20:25:31,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/ab6206b6245a4c00bdc00616fc8bbcb5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ab6206b6245a4c00bdc00616fc8bbcb5 2024-11-17T20:25:31,225 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ab6206b6245a4c00bdc00616fc8bbcb5, entries=150, sequenceid=288, filesize=30.5 K 2024-11-17T20:25:31,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e57c8ee2661e4854a1982e27aac1776d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e57c8ee2661e4854a1982e27aac1776d 2024-11-17T20:25:31,233 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e57c8ee2661e4854a1982e27aac1776d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T20:25:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/914c500cdf0d4b06939f0fc34bbad56e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/914c500cdf0d4b06939f0fc34bbad56e 2024-11-17T20:25:31,238 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/914c500cdf0d4b06939f0fc34bbad56e, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T20:25:31,239 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1077ms, sequenceid=288, compaction requested=false 2024-11-17T20:25:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-17T20:25:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-17T20:25:31,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T20:25:31,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3080 sec 2024-11-17T20:25:31,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.3130 sec 2024-11-17T20:25:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:31,392 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:31,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:31,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170c3b19919ad745168ab47810321e4a59_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875131044/Put/seqid=0 2024-11-17T20:25:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875191411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875191412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875191420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875191421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875191422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742258_1434 (size=12454) 2024-11-17T20:25:31,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875191540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875191540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875191540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875191540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875191540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875191749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875191749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875191750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875191750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875191750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:31,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:31,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:31,854 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170c3b19919ad745168ab47810321e4a59_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170c3b19919ad745168ab47810321e4a59_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:31,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/569861d44a984e868f56bb8c2d6b18d6, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:31,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/569861d44a984e868f56bb8c2d6b18d6 is 175, key is test_row_0/A:col10/1731875131044/Put/seqid=0 2024-11-17T20:25:31,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742259_1435 (size=31255) 2024-11-17T20:25:31,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/569861d44a984e868f56bb8c2d6b18d6 2024-11-17T20:25:31,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9042b235539b48fba094a3585c47142b is 50, key is test_row_0/B:col10/1731875131044/Put/seqid=0 2024-11-17T20:25:31,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742260_1436 (size=12301) 2024-11-17T20:25:31,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9042b235539b48fba094a3585c47142b 2024-11-17T20:25:31,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/7e241049db0b4b0a870b17749afeb5bf is 50, key is test_row_0/C:col10/1731875131044/Put/seqid=0 2024-11-17T20:25:31,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742261_1437 (size=12301) 2024-11-17T20:25:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875192070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875192070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875192070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875192070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875192070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/7e241049db0b4b0a870b17749afeb5bf 2024-11-17T20:25:32,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/569861d44a984e868f56bb8c2d6b18d6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/569861d44a984e868f56bb8c2d6b18d6 2024-11-17T20:25:32,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/569861d44a984e868f56bb8c2d6b18d6, entries=150, sequenceid=316, filesize=30.5 K 2024-11-17T20:25:32,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9042b235539b48fba094a3585c47142b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9042b235539b48fba094a3585c47142b 2024-11-17T20:25:32,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9042b235539b48fba094a3585c47142b, entries=150, sequenceid=316, filesize=12.0 K 2024-11-17T20:25:32,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/7e241049db0b4b0a870b17749afeb5bf as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/7e241049db0b4b0a870b17749afeb5bf 2024-11-17T20:25:32,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/7e241049db0b4b0a870b17749afeb5bf, entries=150, sequenceid=316, filesize=12.0 K 2024-11-17T20:25:32,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 946ms, sequenceid=316, compaction requested=true 2024-11-17T20:25:32,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:32,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:32,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:32,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:32,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:32,339 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:32,342 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:32,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8bc42f47e0ab2de330b6f0f05c9c6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:32,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:32,342 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/A is initiating minor compaction (all files) 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/B is initiating minor compaction (all files) 2024-11-17T20:25:32,343 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/B in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:32,343 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/A in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:32,343 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/39bab2f8027c406a85d256405ab5f5eb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e57c8ee2661e4854a1982e27aac1776d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9042b235539b48fba094a3585c47142b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=36.7 K 2024-11-17T20:25:32,343 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/107a2f18f9b14554a15cd541fc81da9a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ab6206b6245a4c00bdc00616fc8bbcb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/569861d44a984e868f56bb8c2d6b18d6] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=92.2 K 2024-11-17T20:25:32,343 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/107a2f18f9b14554a15cd541fc81da9a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ab6206b6245a4c00bdc00616fc8bbcb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/569861d44a984e868f56bb8c2d6b18d6] 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 39bab2f8027c406a85d256405ab5f5eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731875128707 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 107a2f18f9b14554a15cd541fc81da9a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731875128707 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e57c8ee2661e4854a1982e27aac1776d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731875128840 2024-11-17T20:25:32,343 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab6206b6245a4c00bdc00616fc8bbcb5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731875128840 2024-11-17T20:25:32,344 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9042b235539b48fba094a3585c47142b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731875131044 2024-11-17T20:25:32,344 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 569861d44a984e868f56bb8c2d6b18d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731875131044 2024-11-17T20:25:32,362 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:32,365 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:32,366 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e2907e76012647e293d14ed9ea3bfa0b is 50, key is test_row_0/B:col10/1731875131044/Put/seqid=0 2024-11-17T20:25:32,369 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111768606eb5101e4c1982377b44e3136890_2d8bc42f47e0ab2de330b6f0f05c9c6f store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:32,371 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111768606eb5101e4c1982377b44e3136890_2d8bc42f47e0ab2de330b6f0f05c9c6f, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:32,371 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111768606eb5101e4c1982377b44e3136890_2d8bc42f47e0ab2de330b6f0f05c9c6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:32,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742263_1439 (size=4469) 2024-11-17T20:25:32,385 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#A#compaction#364 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:32,385 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/2d4fdd11e2bd46438e4d2253253b8afd is 175, key is test_row_0/A:col10/1731875131044/Put/seqid=0 2024-11-17T20:25:32,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742262_1438 (size=13051) 2024-11-17T20:25:32,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742264_1440 (size=32005) 2024-11-17T20:25:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:32,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:32,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:32,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117225f8be3ca504b7eb1a575c79f86e577_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875131410/Put/seqid=0 2024-11-17T20:25:32,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742265_1441 (size=14994) 2024-11-17T20:25:32,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:32,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117225f8be3ca504b7eb1a575c79f86e577_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117225f8be3ca504b7eb1a575c79f86e577_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:32,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9ed3780c35dc477cb9d9b5a905867862, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:32,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9ed3780c35dc477cb9d9b5a905867862 is 175, key is test_row_0/A:col10/1731875131410/Put/seqid=0 2024-11-17T20:25:32,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875192657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875192659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875192663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875192664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875192664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742266_1442 (size=39949) 2024-11-17T20:25:32,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875192779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875192779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875192790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875192790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875192790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:32,809 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/e2907e76012647e293d14ed9ea3bfa0b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e2907e76012647e293d14ed9ea3bfa0b 2024-11-17T20:25:32,825 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/B of 2d8bc42f47e0ab2de330b6f0f05c9c6f into e2907e76012647e293d14ed9ea3bfa0b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:32,825 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:32,825 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/B, priority=13, startTime=1731875132338; duration=0sec 2024-11-17T20:25:32,825 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:32,825 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:B 2024-11-17T20:25:32,825 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:32,827 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/2d4fdd11e2bd46438e4d2253253b8afd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2d4fdd11e2bd46438e4d2253253b8afd 2024-11-17T20:25:32,831 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:32,831 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 2d8bc42f47e0ab2de330b6f0f05c9c6f/C is initiating minor compaction (all files) 2024-11-17T20:25:32,831 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8bc42f47e0ab2de330b6f0f05c9c6f/C in TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:32,831 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/44771275df3b4966b49e7936655a07a3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/914c500cdf0d4b06939f0fc34bbad56e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/7e241049db0b4b0a870b17749afeb5bf] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp, totalSize=36.7 K 2024-11-17T20:25:32,832 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 44771275df3b4966b49e7936655a07a3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731875128707 2024-11-17T20:25:32,833 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 914c500cdf0d4b06939f0fc34bbad56e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731875128840 2024-11-17T20:25:32,834 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e241049db0b4b0a870b17749afeb5bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731875131044 2024-11-17T20:25:32,838 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/A of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 2d4fdd11e2bd46438e4d2253253b8afd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:32,838 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:32,838 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/A, priority=13, startTime=1731875132338; duration=0sec 2024-11-17T20:25:32,842 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:32,842 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:A 2024-11-17T20:25:32,850 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8bc42f47e0ab2de330b6f0f05c9c6f#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:32,850 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/5bc11d8e87224a6c939167ffb7e910cb is 50, key is test_row_0/C:col10/1731875131044/Put/seqid=0 2024-11-17T20:25:32,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742267_1443 (size=13051) 2024-11-17T20:25:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875193000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875193000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875193009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875193009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875193009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9ed3780c35dc477cb9d9b5a905867862 2024-11-17T20:25:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-17T20:25:33,099 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:33,100 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-17T20:25:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T20:25:33,102 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:33,103 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:33,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:33,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/07e32f07bd4d4c768d5bdc08a780b58d is 50, key is test_row_0/B:col10/1731875131410/Put/seqid=0 2024-11-17T20:25:33,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742268_1444 (size=12301) 2024-11-17T20:25:33,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/07e32f07bd4d4c768d5bdc08a780b58d 2024-11-17T20:25:33,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/0e04b2c00fb24e7eae02e2a9bee1d551 is 50, key is test_row_0/C:col10/1731875131410/Put/seqid=0 2024-11-17T20:25:33,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742269_1445 (size=12301) 2024-11-17T20:25:33,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/0e04b2c00fb24e7eae02e2a9bee1d551 2024-11-17T20:25:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T20:25:33,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/9ed3780c35dc477cb9d9b5a905867862 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9ed3780c35dc477cb9d9b5a905867862 2024-11-17T20:25:33,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9ed3780c35dc477cb9d9b5a905867862, entries=200, sequenceid=328, filesize=39.0 K 2024-11-17T20:25:33,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/07e32f07bd4d4c768d5bdc08a780b58d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/07e32f07bd4d4c768d5bdc08a780b58d 2024-11-17T20:25:33,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/07e32f07bd4d4c768d5bdc08a780b58d, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:33,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/0e04b2c00fb24e7eae02e2a9bee1d551 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0e04b2c00fb24e7eae02e2a9bee1d551 2024-11-17T20:25:33,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0e04b2c00fb24e7eae02e2a9bee1d551, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:33,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 634ms, sequenceid=328, compaction requested=false 2024-11-17T20:25:33,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:33,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T20:25:33,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:33,256 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:25:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171f904edcf01b4c05998497ad615c76f4_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875132663/Put/seqid=0 2024-11-17T20:25:33,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742270_1446 (size=12454) 2024-11-17T20:25:33,318 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/5bc11d8e87224a6c939167ffb7e910cb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5bc11d8e87224a6c939167ffb7e910cb 2024-11-17T20:25:33,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:33,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. as already flushing 2024-11-17T20:25:33,325 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8bc42f47e0ab2de330b6f0f05c9c6f/C of 2d8bc42f47e0ab2de330b6f0f05c9c6f into 5bc11d8e87224a6c939167ffb7e910cb(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:33,326 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:33,326 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., storeName=2d8bc42f47e0ab2de330b6f0f05c9c6f/C, priority=13, startTime=1731875132339; duration=0sec 2024-11-17T20:25:33,326 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:33,326 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8bc42f47e0ab2de330b6f0f05c9c6f:C 2024-11-17T20:25:33,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875193337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875193338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875193340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875193341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875193342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T20:25:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875193459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875193459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875193459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875193459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875193461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875193669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875193669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875193670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875193670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875193670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:33,694 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171f904edcf01b4c05998497ad615c76f4_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171f904edcf01b4c05998497ad615c76f4_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5c3b864ea8ad46f3be2919590c28d33f, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5c3b864ea8ad46f3be2919590c28d33f is 175, key is test_row_0/A:col10/1731875132663/Put/seqid=0 2024-11-17T20:25:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T20:25:33,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742271_1447 (size=31255) 2024-11-17T20:25:33,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875193980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875193980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875193980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875193980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875193980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:34,152 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5c3b864ea8ad46f3be2919590c28d33f 2024-11-17T20:25:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9f88b278cb2142babd2113454fdc8385 is 50, key is test_row_0/B:col10/1731875132663/Put/seqid=0 2024-11-17T20:25:34,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742272_1448 (size=12301) 2024-11-17T20:25:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T20:25:34,245 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9f88b278cb2142babd2113454fdc8385 2024-11-17T20:25:34,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/fce02d29ec7d41d5a6141160b562a5e7 is 50, key is test_row_0/C:col10/1731875132663/Put/seqid=0 2024-11-17T20:25:34,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742273_1449 (size=12301) 2024-11-17T20:25:34,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57582 deadline: 1731875194499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57580 deadline: 1731875194499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:34,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57522 deadline: 1731875194499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:34,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57554 deadline: 1731875194499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:34,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57574 deadline: 1731875194500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:34,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:25:34,723 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/fce02d29ec7d41d5a6141160b562a5e7 2024-11-17T20:25:34,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/5c3b864ea8ad46f3be2919590c28d33f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5c3b864ea8ad46f3be2919590c28d33f 2024-11-17T20:25:34,734 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5c3b864ea8ad46f3be2919590c28d33f, entries=150, sequenceid=354, filesize=30.5 K 2024-11-17T20:25:34,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/9f88b278cb2142babd2113454fdc8385 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9f88b278cb2142babd2113454fdc8385 2024-11-17T20:25:34,739 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9f88b278cb2142babd2113454fdc8385, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T20:25:34,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/fce02d29ec7d41d5a6141160b562a5e7 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fce02d29ec7d41d5a6141160b562a5e7 2024-11-17T20:25:34,744 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fce02d29ec7d41d5a6141160b562a5e7, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T20:25:34,745 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 1489ms, sequenceid=354, compaction requested=true 2024-11-17T20:25:34,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:34,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:34,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-17T20:25:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-17T20:25:34,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-17T20:25:34,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6440 sec 2024-11-17T20:25:34,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.6490 sec 2024-11-17T20:25:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T20:25:35,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:35,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-17T20:25:35,254 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:35,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:35,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T20:25:35,282 INFO [Thread-1595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1595. 2024-11-17T20:25:35,282 DEBUG [Thread-1595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,282 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,282 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,285 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-17T20:25:35,285 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,285 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,285 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,288 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-11-17T20:25:35,288 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,288 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,288 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,291 INFO [Thread-1593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1593. 2024-11-17T20:25:35,291 DEBUG [Thread-1593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,291 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,291 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,300 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-17T20:25:35,300 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,300 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,300 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T20:25:35,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T20:25:35,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:35,409 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170dc88caa5f624b198f1978824ee1340e_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_0/A:col10/1731875133338/Put/seqid=0 2024-11-17T20:25:35,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742274_1450 (size=12454) 2024-11-17T20:25:35,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:35,429 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170dc88caa5f624b198f1978824ee1340e_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dc88caa5f624b198f1978824ee1340e_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/a896d6cfff9f41e091e04c96905a0627, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:35,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/a896d6cfff9f41e091e04c96905a0627 is 175, key is test_row_0/A:col10/1731875133338/Put/seqid=0 2024-11-17T20:25:35,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742275_1451 (size=31255) 2024-11-17T20:25:35,442 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/a896d6cfff9f41e091e04c96905a0627 2024-11-17T20:25:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/bc55eabba2bd4169b45cbb5b5e45dc71 is 50, key is test_row_0/B:col10/1731875133338/Put/seqid=0 2024-11-17T20:25:35,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742276_1452 (size=12301) 2024-11-17T20:25:35,458 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/bc55eabba2bd4169b45cbb5b5e45dc71 2024-11-17T20:25:35,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/13f5592577fa4c28ac553a12202301c4 is 50, key is test_row_0/C:col10/1731875133338/Put/seqid=0 2024-11-17T20:25:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742277_1453 (size=12301) 2024-11-17T20:25:35,499 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/13f5592577fa4c28ac553a12202301c4 2024-11-17T20:25:35,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/a896d6cfff9f41e091e04c96905a0627 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/a896d6cfff9f41e091e04c96905a0627 2024-11-17T20:25:35,507 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/a896d6cfff9f41e091e04c96905a0627, entries=150, sequenceid=367, filesize=30.5 K 2024-11-17T20:25:35,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/bc55eabba2bd4169b45cbb5b5e45dc71 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bc55eabba2bd4169b45cbb5b5e45dc71 2024-11-17T20:25:35,511 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bc55eabba2bd4169b45cbb5b5e45dc71, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T20:25:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/13f5592577fa4c28ac553a12202301c4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/13f5592577fa4c28ac553a12202301c4 2024-11-17T20:25:35,517 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/13f5592577fa4c28ac553a12202301c4, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T20:25:35,518 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 109ms, sequenceid=367, compaction requested=true 2024-11-17T20:25:35,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: 2024-11-17T20:25:35,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:35,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-17T20:25:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-17T20:25:35,521 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-17T20:25:35,521 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,521 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,522 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,522 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-11-17T20:25:35,522 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,522 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,522 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,522 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-11-17T20:25:35,522 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,522 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,522 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,523 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-11-17T20:25:35,523 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-17T20:25:35,523 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,523 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:35,523 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,524 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,524 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,524 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T20:25:35,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 265 msec 2024-11-17T20:25:35,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 275 msec 2024-11-17T20:25:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T20:25:35,570 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1278 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3832 rows 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1279 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3830 rows 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1285 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3854 rows 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1295 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3880 rows 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1287 2024-11-17T20:25:35,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3857 rows 2024-11-17T20:25:35,570 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:25:35,570 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:25:35,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:35,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:35,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T20:25:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T20:25:35,574 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875135574"}]},"ts":"1731875135574"} 2024-11-17T20:25:35,578 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T20:25:35,578 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T20:25:35,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:25:35,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, UNASSIGN}] 2024-11-17T20:25:35,581 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, UNASSIGN 2024-11-17T20:25:35,582 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:35,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, UNASSIGN because future has completed 2024-11-17T20:25:35,583 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:25:35,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:25:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T20:25:35,736 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:35,736 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:25:35,736 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 2d8bc42f47e0ab2de330b6f0f05c9c6f, disabling compactions & flushes 2024-11-17T20:25:35,736 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:35,736 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:35,736 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. after waiting 0 ms 2024-11-17T20:25:35,736 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:35,736 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T20:25:35,747 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a83309b2f6384cdb99c69a24ad3e0e1b_2d8bc42f47e0ab2de330b6f0f05c9c6f is 50, key is test_row_1/A:col10/1731875135520/Put/seqid=0 2024-11-17T20:25:35,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742278_1454 (size=9914) 2024-11-17T20:25:35,754 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:35,761 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a83309b2f6384cdb99c69a24ad3e0e1b_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a83309b2f6384cdb99c69a24ad3e0e1b_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:35,763 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b38a42df361e48f480276ed11ad1c7d4, store: [table=TestAcidGuarantees family=A region=2d8bc42f47e0ab2de330b6f0f05c9c6f] 2024-11-17T20:25:35,763 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b38a42df361e48f480276ed11ad1c7d4 is 175, key is test_row_1/A:col10/1731875135520/Put/seqid=0 2024-11-17T20:25:35,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742279_1455 (size=22561) 2024-11-17T20:25:35,775 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b38a42df361e48f480276ed11ad1c7d4 2024-11-17T20:25:35,781 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/032acaa1765c4df58d62ad34c9a83f08 is 50, key is test_row_1/B:col10/1731875135520/Put/seqid=0 2024-11-17T20:25:35,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742280_1456 (size=9857) 2024-11-17T20:25:35,790 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/032acaa1765c4df58d62ad34c9a83f08 2024-11-17T20:25:35,797 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/8af2eb38b6114023acd54bca6b92303d is 50, key is test_row_1/C:col10/1731875135520/Put/seqid=0 2024-11-17T20:25:35,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742281_1457 (size=9857) 2024-11-17T20:25:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T20:25:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T20:25:36,201 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/8af2eb38b6114023acd54bca6b92303d 2024-11-17T20:25:36,215 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/A/b38a42df361e48f480276ed11ad1c7d4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b38a42df361e48f480276ed11ad1c7d4 2024-11-17T20:25:36,219 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b38a42df361e48f480276ed11ad1c7d4, entries=100, sequenceid=375, filesize=22.0 K 2024-11-17T20:25:36,219 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/B/032acaa1765c4df58d62ad34c9a83f08 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/032acaa1765c4df58d62ad34c9a83f08 2024-11-17T20:25:36,223 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/032acaa1765c4df58d62ad34c9a83f08, entries=100, sequenceid=375, filesize=9.6 K 2024-11-17T20:25:36,224 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/.tmp/C/8af2eb38b6114023acd54bca6b92303d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/8af2eb38b6114023acd54bca6b92303d 2024-11-17T20:25:36,228 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/8af2eb38b6114023acd54bca6b92303d, entries=100, sequenceid=375, filesize=9.6 K 2024-11-17T20:25:36,232 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 493ms, sequenceid=375, compaction requested=true 2024-11-17T20:25:36,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b684e9319701447cab0b04a91d202eb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/81556b13caac4f05888aadf398eb2037, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/72221797b69d482a8cfa13b4bc9c3f5c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/eb1e9514bae44b0eb18251da5af7e552, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5f848d05a5174fd48bbf40cbf210255d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9b09d529b09344c8b66b1a51e053e4ac, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/d2c22745fc2a41babe84022ec293878d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5bd81afe9f22431f8971737e8de4745e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/20ee589a4f3a434faa7df2a70f27b5a5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e142b2db779b46e1abff8915b993f976, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9d9f7397f9fe477db3c1d6a5c8cc689a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ff543dff2d3f459594db7c8bd041687c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/f04f4c159c4c455a984e1ed4f0cade22, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e93dbe0190054506a4b3ec6e9718ada4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c38ec9f0fefa47a6896c0817d7409e6b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c9d5401bd37d43789e360eb6e93cdd7a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/71c37bfc855b40eaaa8a94008ead1b1d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2cbaf64cbe49446185ff319fd1fcdaed, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/362a27477d714b79bcab062f7739c25e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/107a2f18f9b14554a15cd541fc81da9a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ab6206b6245a4c00bdc00616fc8bbcb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/569861d44a984e868f56bb8c2d6b18d6] to archive 2024-11-17T20:25:36,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:36,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b684e9319701447cab0b04a91d202eb5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b684e9319701447cab0b04a91d202eb5 2024-11-17T20:25:36,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/81556b13caac4f05888aadf398eb2037 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/81556b13caac4f05888aadf398eb2037 2024-11-17T20:25:36,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/72221797b69d482a8cfa13b4bc9c3f5c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/72221797b69d482a8cfa13b4bc9c3f5c 2024-11-17T20:25:36,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/eb1e9514bae44b0eb18251da5af7e552 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/eb1e9514bae44b0eb18251da5af7e552 2024-11-17T20:25:36,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5f848d05a5174fd48bbf40cbf210255d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5f848d05a5174fd48bbf40cbf210255d 2024-11-17T20:25:36,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9b09d529b09344c8b66b1a51e053e4ac to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9b09d529b09344c8b66b1a51e053e4ac 2024-11-17T20:25:36,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/d2c22745fc2a41babe84022ec293878d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/d2c22745fc2a41babe84022ec293878d 2024-11-17T20:25:36,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5bd81afe9f22431f8971737e8de4745e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5bd81afe9f22431f8971737e8de4745e 2024-11-17T20:25:36,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/20ee589a4f3a434faa7df2a70f27b5a5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/20ee589a4f3a434faa7df2a70f27b5a5 2024-11-17T20:25:36,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e142b2db779b46e1abff8915b993f976 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e142b2db779b46e1abff8915b993f976 2024-11-17T20:25:36,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9d9f7397f9fe477db3c1d6a5c8cc689a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9d9f7397f9fe477db3c1d6a5c8cc689a 2024-11-17T20:25:36,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ff543dff2d3f459594db7c8bd041687c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ff543dff2d3f459594db7c8bd041687c 2024-11-17T20:25:36,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/f04f4c159c4c455a984e1ed4f0cade22 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/f04f4c159c4c455a984e1ed4f0cade22 2024-11-17T20:25:36,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e93dbe0190054506a4b3ec6e9718ada4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/e93dbe0190054506a4b3ec6e9718ada4 2024-11-17T20:25:36,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c38ec9f0fefa47a6896c0817d7409e6b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c38ec9f0fefa47a6896c0817d7409e6b 2024-11-17T20:25:36,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c9d5401bd37d43789e360eb6e93cdd7a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/c9d5401bd37d43789e360eb6e93cdd7a 2024-11-17T20:25:36,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/71c37bfc855b40eaaa8a94008ead1b1d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/71c37bfc855b40eaaa8a94008ead1b1d 2024-11-17T20:25:36,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2cbaf64cbe49446185ff319fd1fcdaed to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2cbaf64cbe49446185ff319fd1fcdaed 2024-11-17T20:25:36,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/362a27477d714b79bcab062f7739c25e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/362a27477d714b79bcab062f7739c25e 2024-11-17T20:25:36,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/107a2f18f9b14554a15cd541fc81da9a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/107a2f18f9b14554a15cd541fc81da9a 2024-11-17T20:25:36,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ab6206b6245a4c00bdc00616fc8bbcb5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/ab6206b6245a4c00bdc00616fc8bbcb5 2024-11-17T20:25:36,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/569861d44a984e868f56bb8c2d6b18d6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/569861d44a984e868f56bb8c2d6b18d6 2024-11-17T20:25:36,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/c2ddd2f130fb4cfb9ec7dd78388fb42b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/544c2bf1df594acfbf55f672260ddc10, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/0c16fee46d7044db93b86da37426b7ae, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e17d72b738d842d384da8e89ea6598b4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/740f82a04d164200b2f4c77093c021a1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/2fbfae8de4b6430c874862e102fb29a8, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/73960f5e7f994c3da4dbd990dd3b8404, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ca2f04616a434fc2b0a88e9647f47d1d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/79a762673a4a4fa8a5c65d55d7e00314, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/b79e311edc984355952d56abab5ebb32, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ab673929c4ab4c5e8ddf9e5d0956ab91, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/87a7d22c649145da9e5bb431a246b0cd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/a53403b906da4edf89b44fd1c7d59556, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/fed208ef68804d799d93a30d57eed024, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/5ef66b894a864cc38007069e817681ff, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bde65388c49845f1b5ed7afe2a85ca3e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/dbd1f37381cc45c085ee350e5fb3095e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9588eaa2b2ed43c082711050b8ed65e7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/39bab2f8027c406a85d256405ab5f5eb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/3fb6a66293764c8381a5579794c88ba4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e57c8ee2661e4854a1982e27aac1776d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9042b235539b48fba094a3585c47142b] to archive 2024-11-17T20:25:36,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:36,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/c2ddd2f130fb4cfb9ec7dd78388fb42b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/c2ddd2f130fb4cfb9ec7dd78388fb42b 2024-11-17T20:25:36,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/544c2bf1df594acfbf55f672260ddc10 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/544c2bf1df594acfbf55f672260ddc10 2024-11-17T20:25:36,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/0c16fee46d7044db93b86da37426b7ae to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/0c16fee46d7044db93b86da37426b7ae 2024-11-17T20:25:36,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e17d72b738d842d384da8e89ea6598b4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e17d72b738d842d384da8e89ea6598b4 2024-11-17T20:25:36,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/740f82a04d164200b2f4c77093c021a1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/740f82a04d164200b2f4c77093c021a1 2024-11-17T20:25:36,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/2fbfae8de4b6430c874862e102fb29a8 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/2fbfae8de4b6430c874862e102fb29a8 2024-11-17T20:25:36,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/73960f5e7f994c3da4dbd990dd3b8404 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/73960f5e7f994c3da4dbd990dd3b8404 2024-11-17T20:25:36,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ca2f04616a434fc2b0a88e9647f47d1d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ca2f04616a434fc2b0a88e9647f47d1d 2024-11-17T20:25:36,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/79a762673a4a4fa8a5c65d55d7e00314 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/79a762673a4a4fa8a5c65d55d7e00314 2024-11-17T20:25:36,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/b79e311edc984355952d56abab5ebb32 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/b79e311edc984355952d56abab5ebb32 2024-11-17T20:25:36,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ab673929c4ab4c5e8ddf9e5d0956ab91 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/ab673929c4ab4c5e8ddf9e5d0956ab91 2024-11-17T20:25:36,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/87a7d22c649145da9e5bb431a246b0cd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/87a7d22c649145da9e5bb431a246b0cd 2024-11-17T20:25:36,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/a53403b906da4edf89b44fd1c7d59556 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/a53403b906da4edf89b44fd1c7d59556 2024-11-17T20:25:36,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/fed208ef68804d799d93a30d57eed024 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/fed208ef68804d799d93a30d57eed024 2024-11-17T20:25:36,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/5ef66b894a864cc38007069e817681ff to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/5ef66b894a864cc38007069e817681ff 2024-11-17T20:25:36,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bde65388c49845f1b5ed7afe2a85ca3e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bde65388c49845f1b5ed7afe2a85ca3e 2024-11-17T20:25:36,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/dbd1f37381cc45c085ee350e5fb3095e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/dbd1f37381cc45c085ee350e5fb3095e 2024-11-17T20:25:36,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9588eaa2b2ed43c082711050b8ed65e7 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9588eaa2b2ed43c082711050b8ed65e7 2024-11-17T20:25:36,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/39bab2f8027c406a85d256405ab5f5eb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/39bab2f8027c406a85d256405ab5f5eb 2024-11-17T20:25:36,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/3fb6a66293764c8381a5579794c88ba4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/3fb6a66293764c8381a5579794c88ba4 2024-11-17T20:25:36,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e57c8ee2661e4854a1982e27aac1776d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e57c8ee2661e4854a1982e27aac1776d 2024-11-17T20:25:36,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9042b235539b48fba094a3585c47142b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9042b235539b48fba094a3585c47142b 2024-11-17T20:25:36,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e21a0454d0d14a559ec78fd5b03390a2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/11a5187f3f6f40418a1f0dbee395764b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/a4cf939a961d4d72b54c226fa2e9784a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0d484eb2a40b48cf9b7c8e509ae72147, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/94d2af67a14340d984e74e7485d721c9, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/1f3209763a9b4979b6fc7672ee744c22, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/f831994850c3463183b0c533ecccddc4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/92c66a3216d54e1e8043599bd498c919, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/595e7b8bb68d49b389273372762dadb0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5d37fc8fbf1e4f51b7e204d99084bf12, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/3de057c8816c4528961837d4a1f2028d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/68fc1f0fccb14594ae97d5031de59ae2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/6b51508091f343be960a646adf60b9b5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/06be9b6225fc4d6791a2e84489684d4e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/52193e67a17e4a3090a597d2eb743e7e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fac30a0d63094b79ab45fa8528dfab59, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e4c736cd0ea64ce19ecaa983404d5764, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/44771275df3b4966b49e7936655a07a3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/acd2db9543e24d7faf88f0b46c27bf1b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/914c500cdf0d4b06939f0fc34bbad56e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/7e241049db0b4b0a870b17749afeb5bf] to archive 2024-11-17T20:25:36,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:36,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e21a0454d0d14a559ec78fd5b03390a2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e21a0454d0d14a559ec78fd5b03390a2 2024-11-17T20:25:36,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/11a5187f3f6f40418a1f0dbee395764b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/11a5187f3f6f40418a1f0dbee395764b 2024-11-17T20:25:36,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/a4cf939a961d4d72b54c226fa2e9784a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/a4cf939a961d4d72b54c226fa2e9784a 2024-11-17T20:25:36,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0d484eb2a40b48cf9b7c8e509ae72147 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0d484eb2a40b48cf9b7c8e509ae72147 2024-11-17T20:25:36,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/94d2af67a14340d984e74e7485d721c9 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/94d2af67a14340d984e74e7485d721c9 2024-11-17T20:25:36,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/1f3209763a9b4979b6fc7672ee744c22 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/1f3209763a9b4979b6fc7672ee744c22 2024-11-17T20:25:36,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/f831994850c3463183b0c533ecccddc4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/f831994850c3463183b0c533ecccddc4 2024-11-17T20:25:36,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/92c66a3216d54e1e8043599bd498c919 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/92c66a3216d54e1e8043599bd498c919 2024-11-17T20:25:36,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/595e7b8bb68d49b389273372762dadb0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/595e7b8bb68d49b389273372762dadb0 2024-11-17T20:25:36,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5d37fc8fbf1e4f51b7e204d99084bf12 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5d37fc8fbf1e4f51b7e204d99084bf12 2024-11-17T20:25:36,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/3de057c8816c4528961837d4a1f2028d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/3de057c8816c4528961837d4a1f2028d 2024-11-17T20:25:36,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/68fc1f0fccb14594ae97d5031de59ae2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/68fc1f0fccb14594ae97d5031de59ae2 2024-11-17T20:25:36,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/6b51508091f343be960a646adf60b9b5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/6b51508091f343be960a646adf60b9b5 2024-11-17T20:25:36,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/06be9b6225fc4d6791a2e84489684d4e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/06be9b6225fc4d6791a2e84489684d4e 2024-11-17T20:25:36,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/52193e67a17e4a3090a597d2eb743e7e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/52193e67a17e4a3090a597d2eb743e7e 2024-11-17T20:25:36,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fac30a0d63094b79ab45fa8528dfab59 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fac30a0d63094b79ab45fa8528dfab59 2024-11-17T20:25:36,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e4c736cd0ea64ce19ecaa983404d5764 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/e4c736cd0ea64ce19ecaa983404d5764 2024-11-17T20:25:36,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/b5a0132a5a4b407ea0b68ee7b1b8a7eb 2024-11-17T20:25:36,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/44771275df3b4966b49e7936655a07a3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/44771275df3b4966b49e7936655a07a3 2024-11-17T20:25:36,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/acd2db9543e24d7faf88f0b46c27bf1b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/acd2db9543e24d7faf88f0b46c27bf1b 2024-11-17T20:25:36,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/914c500cdf0d4b06939f0fc34bbad56e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/914c500cdf0d4b06939f0fc34bbad56e 2024-11-17T20:25:36,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/7e241049db0b4b0a870b17749afeb5bf to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/7e241049db0b4b0a870b17749afeb5bf 2024-11-17T20:25:36,343 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/recovered.edits/378.seqid, newMaxSeqId=378, maxSeqId=4 2024-11-17T20:25:36,343 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. 2024-11-17T20:25:36,343 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 2d8bc42f47e0ab2de330b6f0f05c9c6f: Waiting for close lock at 1731875135736Running coprocessor pre-close hooks at 1731875135736Disabling compacts and flushes for region at 1731875135736Disabling writes for close at 1731875135736Obtaining lock to block concurrent updates at 1731875135736Preparing flush snapshotting stores in 2d8bc42f47e0ab2de330b6f0f05c9c6f at 1731875135736Finished memstore snapshotting TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731875135736Flushing stores of TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f. at 1731875135737 (+1 ms)Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/A: creating writer at 1731875135737Flushing mob file 2d8bc42f47e0ab2de330b6f0f05c9c6f/A: appending metadata at 1731875135747 (+10 ms)Flushing mob file 2d8bc42f47e0ab2de330b6f0f05c9c6f/A: closing flushed file at 1731875135747Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/A: appending metadata at 1731875135763 (+16 ms)Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/A: closing flushed file at 1731875135763Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/B: creating writer at 1731875135778 (+15 ms)Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/B: appending metadata at 1731875135781 (+3 ms)Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/B: closing flushed file at 1731875135781Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/C: creating writer at 1731875135794 (+13 ms)Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/C: appending metadata at 1731875135797 (+3 ms)Flushing 2d8bc42f47e0ab2de330b6f0f05c9c6f/C: closing flushed file at 1731875135797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8bed79e: reopening flushed file at 1731875136213 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1967b0f2: reopening flushed file at 1731875136219 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72f6c81d: reopening flushed file at 1731875136223 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2d8bc42f47e0ab2de330b6f0f05c9c6f in 493ms, sequenceid=375, compaction requested=true at 1731875136232 (+9 ms)Writing region close event to WAL at 1731875136340 (+108 ms)Running coprocessor post-close hooks at 1731875136343 (+3 ms)Closed at 1731875136343 2024-11-17T20:25:36,345 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,345 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2d8bc42f47e0ab2de330b6f0f05c9c6f, regionState=CLOSED 2024-11-17T20:25:36,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:25:36,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T20:25:36,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d8bc42f47e0ab2de330b6f0f05c9c6f, server=0b0b0ae8de03,46555,1731875029029 in 764 msec 2024-11-17T20:25:36,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-17T20:25:36,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8bc42f47e0ab2de330b6f0f05c9c6f, UNASSIGN in 768 msec 2024-11-17T20:25:36,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T20:25:36,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 771 msec 2024-11-17T20:25:36,353 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875136352"}]},"ts":"1731875136352"} 2024-11-17T20:25:36,354 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T20:25:36,354 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T20:25:36,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 783 msec 2024-11-17T20:25:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T20:25:36,710 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:36,710 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T20:25:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:36,712 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T20:25:36,713 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:36,715 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,716 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/recovered.edits] 2024-11-17T20:25:36,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2d4fdd11e2bd46438e4d2253253b8afd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/2d4fdd11e2bd46438e4d2253253b8afd 2024-11-17T20:25:36,719 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5c3b864ea8ad46f3be2919590c28d33f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/5c3b864ea8ad46f3be2919590c28d33f 2024-11-17T20:25:36,720 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9ed3780c35dc477cb9d9b5a905867862 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/9ed3780c35dc477cb9d9b5a905867862 2024-11-17T20:25:36,721 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/a896d6cfff9f41e091e04c96905a0627 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/a896d6cfff9f41e091e04c96905a0627 2024-11-17T20:25:36,722 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b38a42df361e48f480276ed11ad1c7d4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/A/b38a42df361e48f480276ed11ad1c7d4 2024-11-17T20:25:36,724 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/032acaa1765c4df58d62ad34c9a83f08 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/032acaa1765c4df58d62ad34c9a83f08 2024-11-17T20:25:36,725 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/07e32f07bd4d4c768d5bdc08a780b58d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/07e32f07bd4d4c768d5bdc08a780b58d 2024-11-17T20:25:36,725 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9f88b278cb2142babd2113454fdc8385 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/9f88b278cb2142babd2113454fdc8385 2024-11-17T20:25:36,726 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bc55eabba2bd4169b45cbb5b5e45dc71 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/bc55eabba2bd4169b45cbb5b5e45dc71 2024-11-17T20:25:36,727 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e2907e76012647e293d14ed9ea3bfa0b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/B/e2907e76012647e293d14ed9ea3bfa0b 2024-11-17T20:25:36,728 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0e04b2c00fb24e7eae02e2a9bee1d551 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/0e04b2c00fb24e7eae02e2a9bee1d551 2024-11-17T20:25:36,729 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/13f5592577fa4c28ac553a12202301c4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/13f5592577fa4c28ac553a12202301c4 2024-11-17T20:25:36,730 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5bc11d8e87224a6c939167ffb7e910cb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/5bc11d8e87224a6c939167ffb7e910cb 2024-11-17T20:25:36,731 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/8af2eb38b6114023acd54bca6b92303d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/8af2eb38b6114023acd54bca6b92303d 2024-11-17T20:25:36,732 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fce02d29ec7d41d5a6141160b562a5e7 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/C/fce02d29ec7d41d5a6141160b562a5e7 2024-11-17T20:25:36,734 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/recovered.edits/378.seqid to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f/recovered.edits/378.seqid 2024-11-17T20:25:36,735 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,735 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T20:25:36,735 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T20:25:36,736 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T20:25:36,739 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111706445f785918493d8384d5e0ae746c4b_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111706445f785918493d8384d5e0ae746c4b_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,740 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170c3b19919ad745168ab47810321e4a59_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170c3b19919ad745168ab47810321e4a59_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,741 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dc88caa5f624b198f1978824ee1340e_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dc88caa5f624b198f1978824ee1340e_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,742 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111710e34a2923d446bdbbdb03252be547f5_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111710e34a2923d446bdbbdb03252be547f5_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,743 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111716baf974834445f08089d76d76c1e900_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111716baf974834445f08089d76d76c1e900_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,744 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171f904edcf01b4c05998497ad615c76f4_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171f904edcf01b4c05998497ad615c76f4_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,745 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117225f8be3ca504b7eb1a575c79f86e577_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117225f8be3ca504b7eb1a575c79f86e577_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,746 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174393c45a9f274a33a81ffc7ad080b37e_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174393c45a9f274a33a81ffc7ad080b37e_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,749 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117528b40d64a5e47ca961ca1fadf741cb4_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117528b40d64a5e47ca961ca1fadf741cb4_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,750 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117737d62d8969a4bf6996cf55c6b2ba21b_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117737d62d8969a4bf6996cf55c6b2ba21b_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,751 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178a6e1bd84cba4cda8e4355171ca58b2a_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178a6e1bd84cba4cda8e4355171ca58b2a_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,751 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e42484a67904fe091445ab293b09b9e_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e42484a67904fe091445ab293b09b9e_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,752 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e5bb3b4a0c14d2489a2762521bc9a48_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e5bb3b4a0c14d2489a2762521bc9a48_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,753 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111796ad32d9a329422388cdbe53a2ecb0dc_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111796ad32d9a329422388cdbe53a2ecb0dc_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,753 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a83309b2f6384cdb99c69a24ad3e0e1b_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a83309b2f6384cdb99c69a24ad3e0e1b_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,754 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a90d56fe9a51447baaae9d4cb521d085_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a90d56fe9a51447baaae9d4cb521d085_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,755 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ac98d019cfa549bebe71476c6e1bc5a4_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ac98d019cfa549bebe71476c6e1bc5a4_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,755 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bf9b98aa5a5241ba93a628915e12b6fd_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bf9b98aa5a5241ba93a628915e12b6fd_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,756 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c69f0140f2974a8292a09ccd6e5b32f3_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c69f0140f2974a8292a09ccd6e5b32f3_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,757 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c95a9d2c901d4a75a0a4f3c0d5cb2a52_2d8bc42f47e0ab2de330b6f0f05c9c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c95a9d2c901d4a75a0a4f3c0d5cb2a52_2d8bc42f47e0ab2de330b6f0f05c9c6f 2024-11-17T20:25:36,758 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T20:25:36,760 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:36,762 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T20:25:36,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T20:25:36,765 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:36,765 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T20:25:36,765 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731875136765"}]},"ts":"9223372036854775807"} 2024-11-17T20:25:36,767 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T20:25:36,767 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2d8bc42f47e0ab2de330b6f0f05c9c6f, NAME => 'TestAcidGuarantees,,1731875112116.2d8bc42f47e0ab2de330b6f0f05c9c6f.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T20:25:36,767 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T20:25:36,767 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731875136767"}]},"ts":"9223372036854775807"} 2024-11-17T20:25:36,769 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T20:25:36,770 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:36,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-17T20:25:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T20:25:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T20:25:36,820 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:36,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=552 (was 510) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8425 (was 8243) - AvailableMemoryMB LEAK? - 2024-11-17T20:25:36,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=552, ProcessCount=11, AvailableMemoryMB=8425 2024-11-17T20:25:36,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:25:36,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:25:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T20:25:36,842 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T20:25:36,843 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:36,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-17T20:25:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-17T20:25:36,843 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T20:25:36,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742282_1458 (size=959) 2024-11-17T20:25:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-17T20:25:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-17T20:25:37,251 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:25:37,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742283_1459 (size=53) 2024-11-17T20:25:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-17T20:25:37,662 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:25:37,662 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6ca3813db0fd868a85c94bd0134efe1f, disabling compactions & flushes 2024-11-17T20:25:37,662 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:37,662 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:37,662 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. after waiting 0 ms 2024-11-17T20:25:37,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:37,663 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:37,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ca3813db0fd868a85c94bd0134efe1f: Waiting for close lock at 1731875137662Disabling compacts and flushes for region at 1731875137662Disabling writes for close at 1731875137663 (+1 ms)Writing region close event to WAL at 1731875137663Closed at 1731875137663 2024-11-17T20:25:37,664 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T20:25:37,664 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731875137664"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875137664"}]},"ts":"1731875137664"} 2024-11-17T20:25:37,665 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T20:25:37,666 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T20:25:37,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875137666"}]},"ts":"1731875137666"} 2024-11-17T20:25:37,667 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T20:25:37,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, ASSIGN}] 2024-11-17T20:25:37,669 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, ASSIGN 2024-11-17T20:25:37,669 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=false 2024-11-17T20:25:37,820 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=6ca3813db0fd868a85c94bd0134efe1f, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:37,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, ASSIGN because future has completed 2024-11-17T20:25:37,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:25:37,976 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:37,976 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:25:37,976 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,977 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:25:37,977 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,977 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,978 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,979 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca3813db0fd868a85c94bd0134efe1f columnFamilyName A 2024-11-17T20:25:37,979 DEBUG [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-17T20:25:37,979 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] regionserver.HStore(327): Store=6ca3813db0fd868a85c94bd0134efe1f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:37,979 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,980 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca3813db0fd868a85c94bd0134efe1f columnFamilyName B 2024-11-17T20:25:37,980 DEBUG [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:37,980 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] regionserver.HStore(327): Store=6ca3813db0fd868a85c94bd0134efe1f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:37,981 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,981 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca3813db0fd868a85c94bd0134efe1f columnFamilyName C 2024-11-17T20:25:37,981 DEBUG [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:25:37,982 INFO [StoreOpener-6ca3813db0fd868a85c94bd0134efe1f-1 {}] regionserver.HStore(327): Store=6ca3813db0fd868a85c94bd0134efe1f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:25:37,982 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,982 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,983 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,983 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,983 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,984 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:25:37,985 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,986 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:25:37,987 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened 6ca3813db0fd868a85c94bd0134efe1f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72355949, jitterRate=0.07818765938282013}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:25:37,987 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:37,987 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for 6ca3813db0fd868a85c94bd0134efe1f: Running coprocessor pre-open hook at 1731875137977Writing region info on filesystem at 1731875137977Initializing all the Stores at 1731875137977Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875137977Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875137978 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875137978Cleaning up temporary data from old regions at 1731875137983 (+5 ms)Running coprocessor post-open hooks at 1731875137987 (+4 ms)Region opened successfully at 1731875137987 2024-11-17T20:25:37,988 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., pid=130, masterSystemTime=1731875137973 2024-11-17T20:25:37,990 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:37,990 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:37,990 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=6ca3813db0fd868a85c94bd0134efe1f, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:37,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:25:37,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T20:25:37,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 in 171 msec 2024-11-17T20:25:37,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-17T20:25:37,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, ASSIGN in 326 msec 2024-11-17T20:25:37,996 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T20:25:37,996 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875137996"}]},"ts":"1731875137996"} 2024-11-17T20:25:37,997 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T20:25:37,998 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T20:25:37,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-11-17T20:25:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-17T20:25:38,990 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:38,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:38,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:38,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:38,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e9ac9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:38,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bbb147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:38,995 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:38,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:38,998 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T20:25:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,32801,1731875028288 2024-11-17T20:25:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61e31bbb 2024-11-17T20:25:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T20:25:39,000 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T20:25:39,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,002 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24fadb50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447d2018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d79b512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,008 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a3767f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,012 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508202ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,014 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47535e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,015 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cae7d52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,017 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330c257e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181848c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,020 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1280673d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,025 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ca3760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,027 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d68723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5fe217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6ed48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,037 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3835a7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,039 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4348fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,052 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6aef91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,054 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49b1d2d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:25:39,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:25:39,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:25:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:25:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:25:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f96300f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:25:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:25:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:39,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:25:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b74a810, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:25:39,061 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,068 DEBUG [Thread-2065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,069 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:39,069 DEBUG [Thread-2054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,069 DEBUG [Thread-2065 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,069 DEBUG [Thread-2065 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T20:25:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T20:25:39,071 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:39,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,072 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:39,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:39,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,076 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:39,084 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:39,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:25:39,085 DEBUG [Thread-2052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,100 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,133 DEBUG [Thread-2056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,134 DEBUG [Thread-2058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875199117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,136 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,137 DEBUG [Thread-2063 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,137 DEBUG [Thread-2063 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875199134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875199137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,140 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:25:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,141 DEBUG [Thread-2069 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:25:39,141 DEBUG [Thread-2069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:25:39,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,143 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:25:39,143 DEBUG [Thread-2063 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2] 2024-11-17T20:25:39,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/9e06ff800cb94f18b1ed4615da1e0463 is 50, key is test_row_0/A:col10/1731875139076/Put/seqid=0 2024-11-17T20:25:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875199145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875199146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742284_1460 (size=12001) 2024-11-17T20:25:39,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/9e06ff800cb94f18b1ed4615da1e0463 2024-11-17T20:25:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T20:25:39,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ef6d866e4d83432eaa2e40133c8d1a2e is 50, key is test_row_0/B:col10/1731875139076/Put/seqid=0 2024-11-17T20:25:39,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T20:25:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:39,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742285_1461 (size=12001) 2024-11-17T20:25:39,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ef6d866e4d83432eaa2e40133c8d1a2e 2024-11-17T20:25:39,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875199239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875199250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875199250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875199260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875199261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/86bd1a3b326a481c9d86d73dced584d1 is 50, key is test_row_0/C:col10/1731875139076/Put/seqid=0 2024-11-17T20:25:39,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742286_1462 (size=12001) 2024-11-17T20:25:39,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/86bd1a3b326a481c9d86d73dced584d1 2024-11-17T20:25:39,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/9e06ff800cb94f18b1ed4615da1e0463 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9e06ff800cb94f18b1ed4615da1e0463 2024-11-17T20:25:39,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9e06ff800cb94f18b1ed4615da1e0463, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T20:25:39,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ef6d866e4d83432eaa2e40133c8d1a2e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ef6d866e4d83432eaa2e40133c8d1a2e 2024-11-17T20:25:39,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ef6d866e4d83432eaa2e40133c8d1a2e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T20:25:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/86bd1a3b326a481c9d86d73dced584d1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/86bd1a3b326a481c9d86d73dced584d1 2024-11-17T20:25:39,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/86bd1a3b326a481c9d86d73dced584d1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T20:25:39,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6ca3813db0fd868a85c94bd0134efe1f in 273ms, sequenceid=12, compaction requested=false 2024-11-17T20:25:39,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:39,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T20:25:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,377 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T20:25:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T20:25:39,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/f830d8a272c54304a0e07959e228fad3 is 50, key is test_row_0/A:col10/1731875139115/Put/seqid=0 2024-11-17T20:25:39,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742287_1463 (size=12001) 2024-11-17T20:25:39,425 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/f830d8a272c54304a0e07959e228fad3 2024-11-17T20:25:39,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/890267f88bc1475994794ce89a6dc508 is 50, key is test_row_0/B:col10/1731875139115/Put/seqid=0 2024-11-17T20:25:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:39,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:39,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742288_1464 (size=12001) 2024-11-17T20:25:39,486 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/890267f88bc1475994794ce89a6dc508 2024-11-17T20:25:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875199472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875199480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875199480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/215e7166512c4d79bf1c7ae1d56d0024 is 50, key is test_row_0/C:col10/1731875139115/Put/seqid=0 2024-11-17T20:25:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875199487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875199490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742289_1465 (size=12001) 2024-11-17T20:25:39,520 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/215e7166512c4d79bf1c7ae1d56d0024 2024-11-17T20:25:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/f830d8a272c54304a0e07959e228fad3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/f830d8a272c54304a0e07959e228fad3 2024-11-17T20:25:39,532 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/f830d8a272c54304a0e07959e228fad3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T20:25:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/890267f88bc1475994794ce89a6dc508 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/890267f88bc1475994794ce89a6dc508 2024-11-17T20:25:39,541 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/890267f88bc1475994794ce89a6dc508, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T20:25:39,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/215e7166512c4d79bf1c7ae1d56d0024 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/215e7166512c4d79bf1c7ae1d56d0024 2024-11-17T20:25:39,554 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/215e7166512c4d79bf1c7ae1d56d0024, entries=150, sequenceid=38, filesize=11.7 K 2024-11-17T20:25:39,555 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6ca3813db0fd868a85c94bd0134efe1f in 178ms, sequenceid=38, compaction requested=false 2024-11-17T20:25:39,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T20:25:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T20:25:39,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T20:25:39,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 485 msec 2024-11-17T20:25:39,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 491 msec 2024-11-17T20:25:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:39,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:39,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:25:39,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/5bc1ae11e1bd4bb0a97750a0ad555a8f is 50, key is test_row_0/A:col10/1731875139610/Put/seqid=0 2024-11-17T20:25:39,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742290_1466 (size=16681) 2024-11-17T20:25:39,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/5bc1ae11e1bd4bb0a97750a0ad555a8f 2024-11-17T20:25:39,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/95e8257e2dc948b583ecda6eab33179b is 50, key is test_row_0/B:col10/1731875139610/Put/seqid=0 2024-11-17T20:25:39,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742291_1467 (size=12001) 2024-11-17T20:25:39,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/95e8257e2dc948b583ecda6eab33179b 2024-11-17T20:25:39,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/a137f1e4425d497a959475dd1992e399 is 50, key is test_row_0/C:col10/1731875139610/Put/seqid=0 2024-11-17T20:25:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875199673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T20:25:39,700 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:39,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T20:25:39,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875199685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T20:25:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875199692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875199692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,705 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:39,707 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:39,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742292_1468 (size=12001) 2024-11-17T20:25:39,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/a137f1e4425d497a959475dd1992e399 2024-11-17T20:25:39,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/5bc1ae11e1bd4bb0a97750a0ad555a8f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5bc1ae11e1bd4bb0a97750a0ad555a8f 2024-11-17T20:25:39,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5bc1ae11e1bd4bb0a97750a0ad555a8f, entries=250, sequenceid=52, filesize=16.3 K 2024-11-17T20:25:39,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/95e8257e2dc948b583ecda6eab33179b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/95e8257e2dc948b583ecda6eab33179b 2024-11-17T20:25:39,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/95e8257e2dc948b583ecda6eab33179b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T20:25:39,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/a137f1e4425d497a959475dd1992e399 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/a137f1e4425d497a959475dd1992e399 2024-11-17T20:25:39,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/a137f1e4425d497a959475dd1992e399, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T20:25:39,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 6ca3813db0fd868a85c94bd0134efe1f in 169ms, sequenceid=52, compaction requested=true 2024-11-17T20:25:39,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:39,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:39,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:39,780 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:39,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:39,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:39,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:39,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T20:25:39,780 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:39,783 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:39,783 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:39,783 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,783 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ef6d866e4d83432eaa2e40133c8d1a2e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/890267f88bc1475994794ce89a6dc508, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/95e8257e2dc948b583ecda6eab33179b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=35.2 K 2024-11-17T20:25:39,784 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ef6d866e4d83432eaa2e40133c8d1a2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731875139075 2024-11-17T20:25:39,784 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 890267f88bc1475994794ce89a6dc508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731875139110 2024-11-17T20:25:39,785 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e8257e2dc948b583ecda6eab33179b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875139605 2024-11-17T20:25:39,786 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:39,786 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:39,786 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,786 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9e06ff800cb94f18b1ed4615da1e0463, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/f830d8a272c54304a0e07959e228fad3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5bc1ae11e1bd4bb0a97750a0ad555a8f] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=39.7 K 2024-11-17T20:25:39,788 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e06ff800cb94f18b1ed4615da1e0463, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731875139075 2024-11-17T20:25:39,790 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting f830d8a272c54304a0e07959e228fad3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731875139110 2024-11-17T20:25:39,791 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc1ae11e1bd4bb0a97750a0ad555a8f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875139469 2024-11-17T20:25:39,791 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:39,791 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/4f01397ebdec48799309fb21932c4b4c is 50, key is test_row_0/B:col10/1731875139610/Put/seqid=0 2024-11-17T20:25:39,806 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:39,807 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/206c1ad9ee0240b3a727d6a2b29b054d is 50, key is test_row_0/A:col10/1731875139610/Put/seqid=0 2024-11-17T20:25:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:39,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:39,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:25:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T20:25:39,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742293_1469 (size=12104) 2024-11-17T20:25:39,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/4cdaea5bfa6c43bbb2aac0c57d216951 is 50, key is test_row_0/A:col10/1731875139807/Put/seqid=0 2024-11-17T20:25:39,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742294_1470 (size=12104) 2024-11-17T20:25:39,837 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/4f01397ebdec48799309fb21932c4b4c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4f01397ebdec48799309fb21932c4b4c 2024-11-17T20:25:39,845 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 4f01397ebdec48799309fb21932c4b4c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:39,845 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:39,845 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875139780; duration=0sec 2024-11-17T20:25:39,845 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:39,845 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:39,845 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:39,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875199833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875199841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,854 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/206c1ad9ee0240b3a727d6a2b29b054d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/206c1ad9ee0240b3a727d6a2b29b054d 2024-11-17T20:25:39,854 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:39,855 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:39,855 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,855 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/86bd1a3b326a481c9d86d73dced584d1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/215e7166512c4d79bf1c7ae1d56d0024, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/a137f1e4425d497a959475dd1992e399] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=35.2 K 2024-11-17T20:25:39,855 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 86bd1a3b326a481c9d86d73dced584d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731875139075 2024-11-17T20:25:39,855 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 215e7166512c4d79bf1c7ae1d56d0024, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731875139110 2024-11-17T20:25:39,855 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a137f1e4425d497a959475dd1992e399, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875139605 2024-11-17T20:25:39,859 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 206c1ad9ee0240b3a727d6a2b29b054d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:39,859 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:39,859 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875139780; duration=0sec 2024-11-17T20:25:39,859 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:39,859 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:39,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:39,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:39,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875199846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875199847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875199847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,868 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:39,868 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/9425beb3893f4933802d65e24ba8c8d3 is 50, key is test_row_0/C:col10/1731875139610/Put/seqid=0 2024-11-17T20:25:39,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742295_1471 (size=14341) 2024-11-17T20:25:39,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742296_1472 (size=12104) 2024-11-17T20:25:39,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875199950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875199950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875199970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875199971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875199971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T20:25:40,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875200160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875200160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875200190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875200190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875200190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/4cdaea5bfa6c43bbb2aac0c57d216951 2024-11-17T20:25:40,297 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/9425beb3893f4933802d65e24ba8c8d3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9425beb3893f4933802d65e24ba8c8d3 2024-11-17T20:25:40,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1857257196c74c7ea9dd7bbec0d9c191 is 50, key is test_row_0/B:col10/1731875139807/Put/seqid=0 2024-11-17T20:25:40,305 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 9425beb3893f4933802d65e24ba8c8d3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:40,305 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:40,305 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875139780; duration=0sec 2024-11-17T20:25:40,305 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:40,305 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:40,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T20:25:40,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742297_1473 (size=12001) 2024-11-17T20:25:40,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1857257196c74c7ea9dd7bbec0d9c191 2024-11-17T20:25:40,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/0b844f415b234f1e81c2ac159eb45f29 is 50, key is test_row_0/C:col10/1731875139807/Put/seqid=0 2024-11-17T20:25:40,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742298_1474 (size=12001) 2024-11-17T20:25:40,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/0b844f415b234f1e81c2ac159eb45f29 2024-11-17T20:25:40,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/4cdaea5bfa6c43bbb2aac0c57d216951 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4cdaea5bfa6c43bbb2aac0c57d216951 2024-11-17T20:25:40,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4cdaea5bfa6c43bbb2aac0c57d216951, entries=200, sequenceid=77, filesize=14.0 K 2024-11-17T20:25:40,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1857257196c74c7ea9dd7bbec0d9c191 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1857257196c74c7ea9dd7bbec0d9c191 2024-11-17T20:25:40,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1857257196c74c7ea9dd7bbec0d9c191, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T20:25:40,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/0b844f415b234f1e81c2ac159eb45f29 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b844f415b234f1e81c2ac159eb45f29 2024-11-17T20:25:40,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b844f415b234f1e81c2ac159eb45f29, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T20:25:40,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 6ca3813db0fd868a85c94bd0134efe1f in 592ms, sequenceid=77, compaction requested=false 2024-11-17T20:25:40,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:40,475 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:40,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:25:40,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:40,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:40,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/19b9769e59ab4c48aab5b405ab436a26 is 50, key is test_row_0/A:col10/1731875139844/Put/seqid=0 2024-11-17T20:25:40,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742299_1475 (size=14341) 2024-11-17T20:25:40,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/19b9769e59ab4c48aab5b405ab436a26 2024-11-17T20:25:40,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/09d475473f724e78bb2be56188ee6152 is 50, key is test_row_0/B:col10/1731875139844/Put/seqid=0 2024-11-17T20:25:40,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742300_1476 (size=12001) 2024-11-17T20:25:40,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/09d475473f724e78bb2be56188ee6152 2024-11-17T20:25:40,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/4af7850b8fc34ca5ab59d030056ffe0e is 50, key is test_row_0/C:col10/1731875139844/Put/seqid=0 2024-11-17T20:25:40,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742301_1477 (size=12001) 2024-11-17T20:25:40,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/4af7850b8fc34ca5ab59d030056ffe0e 2024-11-17T20:25:40,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/19b9769e59ab4c48aab5b405ab436a26 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/19b9769e59ab4c48aab5b405ab436a26 2024-11-17T20:25:40,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/19b9769e59ab4c48aab5b405ab436a26, entries=200, sequenceid=94, filesize=14.0 K 2024-11-17T20:25:40,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/09d475473f724e78bb2be56188ee6152 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/09d475473f724e78bb2be56188ee6152 2024-11-17T20:25:40,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/09d475473f724e78bb2be56188ee6152, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T20:25:40,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/4af7850b8fc34ca5ab59d030056ffe0e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/4af7850b8fc34ca5ab59d030056ffe0e 2024-11-17T20:25:40,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/4af7850b8fc34ca5ab59d030056ffe0e, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T20:25:40,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 6ca3813db0fd868a85c94bd0134efe1f in 117ms, sequenceid=94, compaction requested=true 2024-11-17T20:25:40,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:40,593 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:40,593 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:40,593 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:40,593 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:40,593 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:40,593 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:40,593 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,593 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,593 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4f01397ebdec48799309fb21932c4b4c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1857257196c74c7ea9dd7bbec0d9c191, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/09d475473f724e78bb2be56188ee6152] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=35.3 K 2024-11-17T20:25:40,593 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/206c1ad9ee0240b3a727d6a2b29b054d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4cdaea5bfa6c43bbb2aac0c57d216951, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/19b9769e59ab4c48aab5b405ab436a26] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=39.8 K 2024-11-17T20:25:40,594 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f01397ebdec48799309fb21932c4b4c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875139605 2024-11-17T20:25:40,594 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 206c1ad9ee0240b3a727d6a2b29b054d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875139605 2024-11-17T20:25:40,594 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1857257196c74c7ea9dd7bbec0d9c191, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875139663 2024-11-17T20:25:40,594 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cdaea5bfa6c43bbb2aac0c57d216951, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875139663 2024-11-17T20:25:40,594 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d475473f724e78bb2be56188ee6152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875139831 2024-11-17T20:25:40,594 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b9769e59ab4c48aab5b405ab436a26, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875139831 2024-11-17T20:25:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:40,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:40,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:40,602 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:40,603 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/326979d40e4d4ab487a74007c72a8544 is 50, key is test_row_0/B:col10/1731875139844/Put/seqid=0 2024-11-17T20:25:40,603 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:40,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/4794e75b9e6e4815bb960de54ebb45a4 is 50, key is test_row_0/A:col10/1731875140597/Put/seqid=0 2024-11-17T20:25:40,603 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/142a2ca1a7de46f2900455688f68f8e0 is 50, key is test_row_0/A:col10/1731875139844/Put/seqid=0 2024-11-17T20:25:40,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742303_1479 (size=16681) 2024-11-17T20:25:40,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742304_1480 (size=12207) 2024-11-17T20:25:40,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742302_1478 (size=12207) 2024-11-17T20:25:40,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875200608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,622 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/326979d40e4d4ab487a74007c72a8544 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/326979d40e4d4ab487a74007c72a8544 2024-11-17T20:25:40,629 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 326979d40e4d4ab487a74007c72a8544(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:40,629 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:40,629 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875140593; duration=0sec 2024-11-17T20:25:40,629 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:40,629 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:40,629 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:40,630 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:40,630 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:40,630 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,630 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9425beb3893f4933802d65e24ba8c8d3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b844f415b234f1e81c2ac159eb45f29, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/4af7850b8fc34ca5ab59d030056ffe0e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=35.3 K 2024-11-17T20:25:40,631 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9425beb3893f4933802d65e24ba8c8d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731875139605 2024-11-17T20:25:40,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:40,632 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b844f415b234f1e81c2ac159eb45f29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875139663 2024-11-17T20:25:40,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:40,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875200618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,635 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af7850b8fc34ca5ab59d030056ffe0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875139831 2024-11-17T20:25:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875200622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,641 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:40,641 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/54076310ebfc432d927ad5c48d1d741a is 50, key is test_row_0/C:col10/1731875139844/Put/seqid=0 2024-11-17T20:25:40,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875200629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875200630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742305_1481 (size=12207) 2024-11-17T20:25:40,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875200729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875200739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875200750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875200750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875200750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:40,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:40,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T20:25:40,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:40,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875200950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875200959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875200969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875200969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875200969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/4794e75b9e6e4815bb960de54ebb45a4 2024-11-17T20:25:41,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/453738b006f44c5ea387ca22beaa83fe is 50, key is test_row_0/B:col10/1731875140597/Put/seqid=0 2024-11-17T20:25:41,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742306_1482 (size=12001) 2024-11-17T20:25:41,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/453738b006f44c5ea387ca22beaa83fe 2024-11-17T20:25:41,020 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/142a2ca1a7de46f2900455688f68f8e0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/142a2ca1a7de46f2900455688f68f8e0 2024-11-17T20:25:41,024 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 142a2ca1a7de46f2900455688f68f8e0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:41,024 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:41,024 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875140592; duration=0sec 2024-11-17T20:25:41,024 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:41,024 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:41,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/d850612078604e8784142d1298542d25 is 50, key is test_row_0/C:col10/1731875140597/Put/seqid=0 2024-11-17T20:25:41,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742307_1483 (size=12001) 2024-11-17T20:25:41,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/d850612078604e8784142d1298542d25 2024-11-17T20:25:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/4794e75b9e6e4815bb960de54ebb45a4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4794e75b9e6e4815bb960de54ebb45a4 2024-11-17T20:25:41,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4794e75b9e6e4815bb960de54ebb45a4, entries=250, sequenceid=120, filesize=16.3 K 2024-11-17T20:25:41,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/453738b006f44c5ea387ca22beaa83fe as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/453738b006f44c5ea387ca22beaa83fe 2024-11-17T20:25:41,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/453738b006f44c5ea387ca22beaa83fe, entries=150, sequenceid=120, filesize=11.7 K 2024-11-17T20:25:41,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/d850612078604e8784142d1298542d25 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d850612078604e8784142d1298542d25 2024-11-17T20:25:41,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d850612078604e8784142d1298542d25, entries=150, sequenceid=120, filesize=11.7 K 2024-11-17T20:25:41,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 6ca3813db0fd868a85c94bd0134efe1f in 446ms, sequenceid=120, compaction requested=false 2024-11-17T20:25:41,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:41,053 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/54076310ebfc432d927ad5c48d1d741a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/54076310ebfc432d927ad5c48d1d741a 2024-11-17T20:25:41,057 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 54076310ebfc432d927ad5c48d1d741a(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:41,057 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:41,057 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875140593; duration=0sec 2024-11-17T20:25:41,057 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:41,057 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:41,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T20:25:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:41,093 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:41,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/6f2162f8b4034695881f99c080e7c657 is 50, key is test_row_0/A:col10/1731875140620/Put/seqid=0 2024-11-17T20:25:41,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742308_1484 (size=12051) 2024-11-17T20:25:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:41,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:41,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875201313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875201314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875201314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875201316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875201317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875201429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875201429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875201439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875201439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875201439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,508 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/6f2162f8b4034695881f99c080e7c657 2024-11-17T20:25:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/838450d561e74ad4a5f785277e92374b is 50, key is test_row_0/B:col10/1731875140620/Put/seqid=0 2024-11-17T20:25:41,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742309_1485 (size=12051) 2024-11-17T20:25:41,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875201640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875201649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875201659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875201659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875201659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T20:25:41,866 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T20:25:41,942 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/838450d561e74ad4a5f785277e92374b 2024-11-17T20:25:41,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/83acdc5a1f3343d1899cdb565ff74c2c is 50, key is test_row_0/C:col10/1731875140620/Put/seqid=0 2024-11-17T20:25:41,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875201949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742310_1486 (size=12051) 2024-11-17T20:25:41,956 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/83acdc5a1f3343d1899cdb565ff74c2c 2024-11-17T20:25:41,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/6f2162f8b4034695881f99c080e7c657 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/6f2162f8b4034695881f99c080e7c657 2024-11-17T20:25:41,963 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/6f2162f8b4034695881f99c080e7c657, entries=150, sequenceid=135, filesize=11.8 K 2024-11-17T20:25:41,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/838450d561e74ad4a5f785277e92374b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/838450d561e74ad4a5f785277e92374b 2024-11-17T20:25:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875201959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:41,967 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/838450d561e74ad4a5f785277e92374b, entries=150, sequenceid=135, filesize=11.8 K 2024-11-17T20:25:41,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/83acdc5a1f3343d1899cdb565ff74c2c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/83acdc5a1f3343d1899cdb565ff74c2c 2024-11-17T20:25:41,971 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/83acdc5a1f3343d1899cdb565ff74c2c, entries=150, sequenceid=135, filesize=11.8 K 2024-11-17T20:25:41,972 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6ca3813db0fd868a85c94bd0134efe1f in 879ms, sequenceid=135, compaction requested=true 2024-11-17T20:25:41,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:41,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:41,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T20:25:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T20:25:41,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T20:25:41,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2670 sec 2024-11-17T20:25:41,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.2750 sec 2024-11-17T20:25:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:41,979 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:41,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:41,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/a3e137c754b94a4083025a4867eeb17d is 50, key is test_row_0/A:col10/1731875141299/Put/seqid=0 2024-11-17T20:25:41,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742311_1487 (size=14541) 2024-11-17T20:25:41,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/a3e137c754b94a4083025a4867eeb17d 2024-11-17T20:25:42,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/37f1e224eace492db7b146c0317438c2 is 50, key is test_row_0/B:col10/1731875141299/Put/seqid=0 2024-11-17T20:25:42,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742312_1488 (size=12151) 2024-11-17T20:25:42,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/37f1e224eace492db7b146c0317438c2 2024-11-17T20:25:42,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/3abd1431489b4b71982bc832db92a751 is 50, key is test_row_0/C:col10/1731875141299/Put/seqid=0 2024-11-17T20:25:42,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742313_1489 (size=12151) 2024-11-17T20:25:42,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/3abd1431489b4b71982bc832db92a751 2024-11-17T20:25:42,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/a3e137c754b94a4083025a4867eeb17d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/a3e137c754b94a4083025a4867eeb17d 2024-11-17T20:25:42,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875202036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/a3e137c754b94a4083025a4867eeb17d, entries=200, sequenceid=160, filesize=14.2 K 2024-11-17T20:25:42,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/37f1e224eace492db7b146c0317438c2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/37f1e224eace492db7b146c0317438c2 2024-11-17T20:25:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875202038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875202039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/37f1e224eace492db7b146c0317438c2, entries=150, sequenceid=160, filesize=11.9 K 2024-11-17T20:25:42,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/3abd1431489b4b71982bc832db92a751 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3abd1431489b4b71982bc832db92a751 2024-11-17T20:25:42,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3abd1431489b4b71982bc832db92a751, entries=150, sequenceid=160, filesize=11.9 K 2024-11-17T20:25:42,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6ca3813db0fd868a85c94bd0134efe1f in 88ms, sequenceid=160, compaction requested=true 2024-11-17T20:25:42,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:42,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:42,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:42,067 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:42,067 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:42,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:42,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:42,069 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:42,069 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:42,069 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:42,069 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/326979d40e4d4ab487a74007c72a8544, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/453738b006f44c5ea387ca22beaa83fe, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/838450d561e74ad4a5f785277e92374b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/37f1e224eace492db7b146c0317438c2] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=47.3 K 2024-11-17T20:25:42,069 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:42,069 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:42,069 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:42,069 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/142a2ca1a7de46f2900455688f68f8e0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4794e75b9e6e4815bb960de54ebb45a4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/6f2162f8b4034695881f99c080e7c657, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/a3e137c754b94a4083025a4867eeb17d] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=54.2 K 2024-11-17T20:25:42,070 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142a2ca1a7de46f2900455688f68f8e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875139831 2024-11-17T20:25:42,070 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 326979d40e4d4ab487a74007c72a8544, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875139831 2024-11-17T20:25:42,071 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4794e75b9e6e4815bb960de54ebb45a4, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731875140532 2024-11-17T20:25:42,071 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 453738b006f44c5ea387ca22beaa83fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731875140532 2024-11-17T20:25:42,071 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 838450d561e74ad4a5f785277e92374b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731875140606 2024-11-17T20:25:42,071 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2162f8b4034695881f99c080e7c657, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731875140606 2024-11-17T20:25:42,071 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f1e224eace492db7b146c0317438c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731875141299 2024-11-17T20:25:42,072 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e137c754b94a4083025a4867eeb17d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731875141299 2024-11-17T20:25:42,090 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:42,090 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/d556ad8f9e5f483e94425a60af0a7e90 is 50, key is test_row_0/A:col10/1731875141299/Put/seqid=0 2024-11-17T20:25:42,098 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:42,098 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/c0a1925070864066a48454c848ed443c is 50, key is test_row_0/B:col10/1731875141299/Put/seqid=0 2024-11-17T20:25:42,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742314_1490 (size=12493) 2024-11-17T20:25:42,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742315_1491 (size=12493) 2024-11-17T20:25:42,122 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/c0a1925070864066a48454c848ed443c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/c0a1925070864066a48454c848ed443c 2024-11-17T20:25:42,130 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into c0a1925070864066a48454c848ed443c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:42,130 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:42,130 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=12, startTime=1731875142067; duration=0sec 2024-11-17T20:25:42,130 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:42,130 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:42,130 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:25:42,131 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:25:42,131 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:42,131 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:42,131 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/54076310ebfc432d927ad5c48d1d741a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d850612078604e8784142d1298542d25, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/83acdc5a1f3343d1899cdb565ff74c2c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3abd1431489b4b71982bc832db92a751] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=47.3 K 2024-11-17T20:25:42,131 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 54076310ebfc432d927ad5c48d1d741a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731875139831 2024-11-17T20:25:42,132 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d850612078604e8784142d1298542d25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731875140532 2024-11-17T20:25:42,132 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 83acdc5a1f3343d1899cdb565ff74c2c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731875140606 2024-11-17T20:25:42,132 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3abd1431489b4b71982bc832db92a751, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731875141299 2024-11-17T20:25:42,148 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:42,148 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/864b8f895d31431996afa05655c5de86 is 50, key is test_row_0/C:col10/1731875141299/Put/seqid=0 2024-11-17T20:25:42,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742316_1492 (size=12493) 2024-11-17T20:25:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:42,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:42,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:25:42,173 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/864b8f895d31431996afa05655c5de86 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/864b8f895d31431996afa05655c5de86 2024-11-17T20:25:42,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/212c21e3ba3d4f1c994e597da1df8be0 is 50, key is test_row_0/A:col10/1731875142038/Put/seqid=0 2024-11-17T20:25:42,178 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 864b8f895d31431996afa05655c5de86(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:42,178 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:42,178 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=12, startTime=1731875142068; duration=0sec 2024-11-17T20:25:42,179 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:42,179 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:42,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742317_1493 (size=14541) 2024-11-17T20:25:42,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/212c21e3ba3d4f1c994e597da1df8be0 2024-11-17T20:25:42,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/338fb57e91674a54b5c23fe9f24c4119 is 50, key is test_row_0/B:col10/1731875142038/Put/seqid=0 2024-11-17T20:25:42,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742318_1494 (size=12151) 2024-11-17T20:25:42,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875202277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875202278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875202280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875202399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875202399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875202399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875202459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875202480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,513 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/d556ad8f9e5f483e94425a60af0a7e90 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/d556ad8f9e5f483e94425a60af0a7e90 2024-11-17T20:25:42,522 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into d556ad8f9e5f483e94425a60af0a7e90(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:42,522 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:42,522 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=12, startTime=1731875142067; duration=0sec 2024-11-17T20:25:42,522 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:42,522 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:42,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/338fb57e91674a54b5c23fe9f24c4119 2024-11-17T20:25:42,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5438bd46ca77434d8b9d506708712dd2 is 50, key is test_row_0/C:col10/1731875142038/Put/seqid=0 2024-11-17T20:25:42,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742319_1495 (size=12151) 2024-11-17T20:25:42,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875202620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875202620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875202620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875202940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875202940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875202940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5438bd46ca77434d8b9d506708712dd2 2024-11-17T20:25:43,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/212c21e3ba3d4f1c994e597da1df8be0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/212c21e3ba3d4f1c994e597da1df8be0 2024-11-17T20:25:43,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/212c21e3ba3d4f1c994e597da1df8be0, entries=200, sequenceid=173, filesize=14.2 K 2024-11-17T20:25:43,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/338fb57e91674a54b5c23fe9f24c4119 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/338fb57e91674a54b5c23fe9f24c4119 2024-11-17T20:25:43,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/338fb57e91674a54b5c23fe9f24c4119, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T20:25:43,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5438bd46ca77434d8b9d506708712dd2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5438bd46ca77434d8b9d506708712dd2 2024-11-17T20:25:43,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5438bd46ca77434d8b9d506708712dd2, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T20:25:43,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6ca3813db0fd868a85c94bd0134efe1f in 871ms, sequenceid=173, compaction requested=false 2024-11-17T20:25:43,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:43,452 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:43,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:25:43,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/cf8ad4d2342f434aa9b3a439c3d3a2ec is 50, key is test_row_0/A:col10/1731875142278/Put/seqid=0 2024-11-17T20:25:43,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742320_1496 (size=14541) 2024-11-17T20:25:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875203466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875203467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875203469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875203473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,478 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875203499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,507 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875203579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875203589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875203590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875203789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875203799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875203799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T20:25:43,859 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:43,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-17T20:25:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T20:25:43,861 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:43,862 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:43,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:43,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/cf8ad4d2342f434aa9b3a439c3d3a2ec 2024-11-17T20:25:43,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/dbb2811229b64094a01a7e9fbcf29b39 is 50, key is test_row_0/B:col10/1731875142278/Put/seqid=0 2024-11-17T20:25:43,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742321_1497 (size=12151) 2024-11-17T20:25:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T20:25:44,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T20:25:44,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:44,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875204099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875204109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875204109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:44,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T20:25:44,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:44,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T20:25:44,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/dbb2811229b64094a01a7e9fbcf29b39 2024-11-17T20:25:44,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/23daf64a98394f0f8dcee24efbdc504e is 50, key is test_row_0/C:col10/1731875142278/Put/seqid=0 2024-11-17T20:25:44,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742322_1498 (size=12151) 2024-11-17T20:25:44,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T20:25:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T20:25:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T20:25:44,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875204609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:44,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875204619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:44,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875204619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:44,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T20:25:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:44,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/23daf64a98394f0f8dcee24efbdc504e 2024-11-17T20:25:44,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/cf8ad4d2342f434aa9b3a439c3d3a2ec as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/cf8ad4d2342f434aa9b3a439c3d3a2ec 2024-11-17T20:25:44,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/cf8ad4d2342f434aa9b3a439c3d3a2ec, entries=200, sequenceid=200, filesize=14.2 K 2024-11-17T20:25:44,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/dbb2811229b64094a01a7e9fbcf29b39 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/dbb2811229b64094a01a7e9fbcf29b39 2024-11-17T20:25:44,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/dbb2811229b64094a01a7e9fbcf29b39, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T20:25:44,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/23daf64a98394f0f8dcee24efbdc504e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/23daf64a98394f0f8dcee24efbdc504e 2024-11-17T20:25:44,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/23daf64a98394f0f8dcee24efbdc504e, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T20:25:44,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6ca3813db0fd868a85c94bd0134efe1f in 1240ms, sequenceid=200, compaction requested=true 2024-11-17T20:25:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:44,692 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:44,692 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:44,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:44,693 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:44,693 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:44,693 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:44,693 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:44,693 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,693 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,693 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/d556ad8f9e5f483e94425a60af0a7e90, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/212c21e3ba3d4f1c994e597da1df8be0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/cf8ad4d2342f434aa9b3a439c3d3a2ec] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=40.6 K 2024-11-17T20:25:44,693 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/c0a1925070864066a48454c848ed443c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/338fb57e91674a54b5c23fe9f24c4119, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/dbb2811229b64094a01a7e9fbcf29b39] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=35.9 K 2024-11-17T20:25:44,694 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c0a1925070864066a48454c848ed443c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731875141299 2024-11-17T20:25:44,694 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d556ad8f9e5f483e94425a60af0a7e90, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731875141299 2024-11-17T20:25:44,694 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 212c21e3ba3d4f1c994e597da1df8be0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731875141995 2024-11-17T20:25:44,694 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 338fb57e91674a54b5c23fe9f24c4119, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731875141995 2024-11-17T20:25:44,694 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf8ad4d2342f434aa9b3a439c3d3a2ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875142230 2024-11-17T20:25:44,694 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb2811229b64094a01a7e9fbcf29b39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875142230 2024-11-17T20:25:44,703 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:44,704 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c17494f2a48d4d1085f3dfba4570c418 is 50, key is test_row_0/A:col10/1731875142278/Put/seqid=0 2024-11-17T20:25:44,710 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:44,711 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1162cd367f514375a93d008102feac53 is 50, key is test_row_0/B:col10/1731875142278/Put/seqid=0 2024-11-17T20:25:44,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742323_1499 (size=12595) 2024-11-17T20:25:44,719 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c17494f2a48d4d1085f3dfba4570c418 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c17494f2a48d4d1085f3dfba4570c418 2024-11-17T20:25:44,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742324_1500 (size=12595) 2024-11-17T20:25:44,731 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into c17494f2a48d4d1085f3dfba4570c418(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:44,731 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:44,731 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875144692; duration=0sec 2024-11-17T20:25:44,731 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:44,731 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:44,731 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:44,731 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1162cd367f514375a93d008102feac53 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1162cd367f514375a93d008102feac53 2024-11-17T20:25:44,732 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:44,732 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:44,732 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,732 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/864b8f895d31431996afa05655c5de86, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5438bd46ca77434d8b9d506708712dd2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/23daf64a98394f0f8dcee24efbdc504e] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=35.9 K 2024-11-17T20:25:44,733 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864b8f895d31431996afa05655c5de86, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731875141299 2024-11-17T20:25:44,733 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5438bd46ca77434d8b9d506708712dd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731875141995 2024-11-17T20:25:44,733 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23daf64a98394f0f8dcee24efbdc504e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875142230 2024-11-17T20:25:44,737 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 1162cd367f514375a93d008102feac53(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:44,737 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:44,738 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875144692; duration=0sec 2024-11-17T20:25:44,738 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:44,738 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:44,741 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:44,742 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5a50285588c542319438023f56e9483d is 50, key is test_row_0/C:col10/1731875142278/Put/seqid=0 2024-11-17T20:25:44,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742325_1501 (size=12595) 2024-11-17T20:25:44,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T20:25:44,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:44,777 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:25:44,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/5556d9e4ee8444449095b7bf2b3f4993 is 50, key is test_row_0/A:col10/1731875143472/Put/seqid=0 2024-11-17T20:25:44,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742326_1502 (size=12151) 2024-11-17T20:25:44,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T20:25:45,152 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5a50285588c542319438023f56e9483d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5a50285588c542319438023f56e9483d 2024-11-17T20:25:45,155 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 5a50285588c542319438023f56e9483d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:45,155 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:45,155 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875144692; duration=0sec 2024-11-17T20:25:45,155 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:45,155 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:45,198 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/5556d9e4ee8444449095b7bf2b3f4993 2024-11-17T20:25:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/3163b3463e3e4544a287da8b1979ebb5 is 50, key is test_row_0/B:col10/1731875143472/Put/seqid=0 2024-11-17T20:25:45,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742327_1503 (size=12151) 2024-11-17T20:25:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:45,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875205570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875205573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,619 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/3163b3463e3e4544a287da8b1979ebb5 2024-11-17T20:25:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875205620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,622 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/bf2219fb3c9546738d88efed734f0575 is 50, key is test_row_0/C:col10/1731875143472/Put/seqid=0 2024-11-17T20:25:45,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742328_1504 (size=12151) 2024-11-17T20:25:45,628 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/bf2219fb3c9546738d88efed734f0575 2024-11-17T20:25:45,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/5556d9e4ee8444449095b7bf2b3f4993 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5556d9e4ee8444449095b7bf2b3f4993 2024-11-17T20:25:45,634 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5556d9e4ee8444449095b7bf2b3f4993, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T20:25:45,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/3163b3463e3e4544a287da8b1979ebb5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3163b3463e3e4544a287da8b1979ebb5 2024-11-17T20:25:45,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875205629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,636 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,637 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3163b3463e3e4544a287da8b1979ebb5, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T20:25:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/bf2219fb3c9546738d88efed734f0575 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/bf2219fb3c9546738d88efed734f0575 2024-11-17T20:25:45,640 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/bf2219fb3c9546738d88efed734f0575, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T20:25:45,641 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6ca3813db0fd868a85c94bd0134efe1f in 864ms, sequenceid=213, compaction requested=false 2024-11-17T20:25:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-17T20:25:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-17T20:25:45,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T20:25:45,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-11-17T20:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:45,644 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:45,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-17T20:25:45,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.7840 sec 2024-11-17T20:25:45,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/7ae26269aeef45059f9cfba04249f2c5 is 50, key is test_row_0/A:col10/1731875145572/Put/seqid=0 2024-11-17T20:25:45,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742329_1505 (size=14541) 2024-11-17T20:25:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875205675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875205679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875205690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875205789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875205889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:45,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875205900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875205999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T20:25:46,010 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:46,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-17T20:25:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T20:25:46,012 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:46,012 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:46,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:46,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/7ae26269aeef45059f9cfba04249f2c5 2024-11-17T20:25:46,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/aa00dc5efe894b489af8cec2d630b88b is 50, key is test_row_0/B:col10/1731875145572/Put/seqid=0 2024-11-17T20:25:46,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742330_1506 (size=12151) 2024-11-17T20:25:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T20:25:46,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T20:25:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875206199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875206209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875206309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T20:25:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T20:25:46,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/aa00dc5efe894b489af8cec2d630b88b 2024-11-17T20:25:46,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/2efefc6f15944877be05091c7a9d1f8c is 50, key is test_row_0/C:col10/1731875145572/Put/seqid=0 2024-11-17T20:25:46,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742331_1507 (size=12151) 2024-11-17T20:25:46,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T20:25:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:46,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/2efefc6f15944877be05091c7a9d1f8c 2024-11-17T20:25:46,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/7ae26269aeef45059f9cfba04249f2c5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ae26269aeef45059f9cfba04249f2c5 2024-11-17T20:25:46,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ae26269aeef45059f9cfba04249f2c5, entries=200, sequenceid=240, filesize=14.2 K 2024-11-17T20:25:46,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/aa00dc5efe894b489af8cec2d630b88b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/aa00dc5efe894b489af8cec2d630b88b 2024-11-17T20:25:46,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/aa00dc5efe894b489af8cec2d630b88b, entries=150, sequenceid=240, filesize=11.9 K 2024-11-17T20:25:46,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/2efefc6f15944877be05091c7a9d1f8c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/2efefc6f15944877be05091c7a9d1f8c 2024-11-17T20:25:46,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/2efefc6f15944877be05091c7a9d1f8c, entries=150, sequenceid=240, filesize=11.9 K 2024-11-17T20:25:46,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6ca3813db0fd868a85c94bd0134efe1f in 840ms, sequenceid=240, compaction requested=true 2024-11-17T20:25:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:46,484 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:46,484 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:46,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:46,486 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,486 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,486 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c17494f2a48d4d1085f3dfba4570c418, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5556d9e4ee8444449095b7bf2b3f4993, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ae26269aeef45059f9cfba04249f2c5] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=38.4 K 2024-11-17T20:25:46,486 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1162cd367f514375a93d008102feac53, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3163b3463e3e4544a287da8b1979ebb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/aa00dc5efe894b489af8cec2d630b88b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.0 K 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c17494f2a48d4d1085f3dfba4570c418, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875142230 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1162cd367f514375a93d008102feac53, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875142230 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5556d9e4ee8444449095b7bf2b3f4993, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731875143454 2024-11-17T20:25:46,486 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3163b3463e3e4544a287da8b1979ebb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731875143454 2024-11-17T20:25:46,487 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ae26269aeef45059f9cfba04249f2c5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731875145557 2024-11-17T20:25:46,487 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting aa00dc5efe894b489af8cec2d630b88b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731875145557 2024-11-17T20:25:46,492 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:46,492 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:46,492 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/4fdf6683e0dd494a86f3d48a2b31870c is 50, key is test_row_0/B:col10/1731875145572/Put/seqid=0 2024-11-17T20:25:46,492 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/485182ff03934a32934e001cd0e8a8ec is 50, key is test_row_0/A:col10/1731875145572/Put/seqid=0 2024-11-17T20:25:46,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742332_1508 (size=12697) 2024-11-17T20:25:46,513 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/4fdf6683e0dd494a86f3d48a2b31870c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4fdf6683e0dd494a86f3d48a2b31870c 2024-11-17T20:25:46,517 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 4fdf6683e0dd494a86f3d48a2b31870c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:46,517 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:46,517 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875146484; duration=0sec 2024-11-17T20:25:46,517 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:46,517 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:46,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742333_1509 (size=12697) 2024-11-17T20:25:46,517 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:46,518 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:46,518 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:46,518 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,519 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5a50285588c542319438023f56e9483d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/bf2219fb3c9546738d88efed734f0575, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/2efefc6f15944877be05091c7a9d1f8c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.0 K 2024-11-17T20:25:46,519 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a50285588c542319438023f56e9483d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875142230 2024-11-17T20:25:46,519 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2219fb3c9546738d88efed734f0575, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731875143454 2024-11-17T20:25:46,519 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efefc6f15944877be05091c7a9d1f8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731875145557 2024-11-17T20:25:46,523 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/485182ff03934a32934e001cd0e8a8ec as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/485182ff03934a32934e001cd0e8a8ec 2024-11-17T20:25:46,526 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:46,526 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/b9f812895cfa4a6cb374be6cef66d79a is 50, key is test_row_0/C:col10/1731875145572/Put/seqid=0 2024-11-17T20:25:46,527 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 485182ff03934a32934e001cd0e8a8ec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:46,527 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:46,527 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875146484; duration=0sec 2024-11-17T20:25:46,527 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:46,527 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:46,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742334_1510 (size=12697) 2024-11-17T20:25:46,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T20:25:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:46,622 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T20:25:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/0ae6db38f079437ca2aae1c2515704b0 is 50, key is test_row_0/A:col10/1731875145664/Put/seqid=0 2024-11-17T20:25:46,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742335_1511 (size=12151) 2024-11-17T20:25:46,632 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/0ae6db38f079437ca2aae1c2515704b0 2024-11-17T20:25:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T20:25:46,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/3f326d7a7db84b589721d7d8e6b70149 is 50, key is test_row_0/B:col10/1731875145664/Put/seqid=0 2024-11-17T20:25:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742336_1512 (size=12151) 2024-11-17T20:25:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:46,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875206788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875206790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875206819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875206899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875206899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:46,936 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/b9f812895cfa4a6cb374be6cef66d79a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b9f812895cfa4a6cb374be6cef66d79a 2024-11-17T20:25:46,939 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into b9f812895cfa4a6cb374be6cef66d79a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:46,939 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:46,939 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875146484; duration=0sec 2024-11-17T20:25:46,939 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:46,939 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:47,049 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/3f326d7a7db84b589721d7d8e6b70149 2024-11-17T20:25:47,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/7f490fd05a2c45f7b331dc755398a40b is 50, key is test_row_0/C:col10/1731875145664/Put/seqid=0 2024-11-17T20:25:47,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742337_1513 (size=12151) 2024-11-17T20:25:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875207109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875207109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T20:25:47,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T20:25:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875207419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875207419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,459 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/7f490fd05a2c45f7b331dc755398a40b 2024-11-17T20:25:47,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/0ae6db38f079437ca2aae1c2515704b0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0ae6db38f079437ca2aae1c2515704b0 2024-11-17T20:25:47,464 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0ae6db38f079437ca2aae1c2515704b0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-17T20:25:47,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/3f326d7a7db84b589721d7d8e6b70149 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3f326d7a7db84b589721d7d8e6b70149 2024-11-17T20:25:47,468 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3f326d7a7db84b589721d7d8e6b70149, entries=150, sequenceid=252, filesize=11.9 K 2024-11-17T20:25:47,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/7f490fd05a2c45f7b331dc755398a40b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7f490fd05a2c45f7b331dc755398a40b 2024-11-17T20:25:47,471 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7f490fd05a2c45f7b331dc755398a40b, entries=150, sequenceid=252, filesize=11.9 K 2024-11-17T20:25:47,472 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 6ca3813db0fd868a85c94bd0134efe1f in 850ms, sequenceid=252, compaction requested=false 2024-11-17T20:25:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-17T20:25:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-17T20:25:47,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T20:25:47,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-11-17T20:25:47,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4640 sec 2024-11-17T20:25:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:47,632 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:47,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T20:25:47,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3c096b1e0c5242a7b8a89b539a21c889 is 50, key is test_row_0/A:col10/1731875146771/Put/seqid=0 2024-11-17T20:25:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742338_1514 (size=14741) 2024-11-17T20:25:47,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875207659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,666 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875207662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875207779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875207829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,834 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875207930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875207939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875207989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:48,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3c096b1e0c5242a7b8a89b539a21c889 2024-11-17T20:25:48,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/16fba09dacaf4dd796cbb0d07379ed77 is 50, key is test_row_0/B:col10/1731875146771/Put/seqid=0 2024-11-17T20:25:48,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742339_1515 (size=12301) 2024-11-17T20:25:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T20:25:48,160 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:48,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-17T20:25:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T20:25:48,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:48,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:48,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T20:25:48,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875208299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:48,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T20:25:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/16fba09dacaf4dd796cbb0d07379ed77 2024-11-17T20:25:48,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/1b39da7844b849c8a1c0d10ee8418a93 is 50, key is test_row_0/C:col10/1731875146771/Put/seqid=0 2024-11-17T20:25:48,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742340_1516 (size=12301) 2024-11-17T20:25:48,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T20:25:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T20:25:48,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T20:25:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T20:25:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T20:25:48,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875208809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:48,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/1b39da7844b849c8a1c0d10ee8418a93 2024-11-17T20:25:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3c096b1e0c5242a7b8a89b539a21c889 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3c096b1e0c5242a7b8a89b539a21c889 2024-11-17T20:25:48,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3c096b1e0c5242a7b8a89b539a21c889, entries=200, sequenceid=280, filesize=14.4 K 2024-11-17T20:25:48,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/16fba09dacaf4dd796cbb0d07379ed77 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/16fba09dacaf4dd796cbb0d07379ed77 2024-11-17T20:25:48,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/16fba09dacaf4dd796cbb0d07379ed77, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T20:25:48,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/1b39da7844b849c8a1c0d10ee8418a93 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/1b39da7844b849c8a1c0d10ee8418a93 2024-11-17T20:25:48,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/1b39da7844b849c8a1c0d10ee8418a93, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T20:25:48,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 6ca3813db0fd868a85c94bd0134efe1f in 1239ms, sequenceid=280, compaction requested=true 2024-11-17T20:25:48,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:48,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:48,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:48,871 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:48,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:48,872 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:48,872 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:48,872 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:48,873 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:48,873 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:48,873 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,873 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,873 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/485182ff03934a32934e001cd0e8a8ec, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0ae6db38f079437ca2aae1c2515704b0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3c096b1e0c5242a7b8a89b539a21c889] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=38.7 K 2024-11-17T20:25:48,873 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4fdf6683e0dd494a86f3d48a2b31870c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3f326d7a7db84b589721d7d8e6b70149, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/16fba09dacaf4dd796cbb0d07379ed77] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.3 K 2024-11-17T20:25:48,873 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485182ff03934a32934e001cd0e8a8ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731875145557 2024-11-17T20:25:48,873 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fdf6683e0dd494a86f3d48a2b31870c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731875145557 2024-11-17T20:25:48,873 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae6db38f079437ca2aae1c2515704b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731875145664 2024-11-17T20:25:48,873 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f326d7a7db84b589721d7d8e6b70149, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731875145664 2024-11-17T20:25:48,874 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c096b1e0c5242a7b8a89b539a21c889, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731875146771 2024-11-17T20:25:48,874 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 16fba09dacaf4dd796cbb0d07379ed77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731875146771 2024-11-17T20:25:48,879 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:48,879 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:48,880 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/afd6681a39414a2191392895b9cb7c50 is 50, key is test_row_0/B:col10/1731875146771/Put/seqid=0 2024-11-17T20:25:48,880 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/683550f821ac4c1a85b53a4271a8b827 is 50, key is test_row_0/A:col10/1731875146771/Put/seqid=0 2024-11-17T20:25:48,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742341_1517 (size=12949) 2024-11-17T20:25:48,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742342_1518 (size=12949) 2024-11-17T20:25:48,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T20:25:48,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:48,925 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-17T20:25:48,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c1b09f49311f4f679bb33da69ba0d194 is 50, key is test_row_0/A:col10/1731875147661/Put/seqid=0 2024-11-17T20:25:48,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742343_1519 (size=12301) 2024-11-17T20:25:48,931 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c1b09f49311f4f679bb33da69ba0d194 2024-11-17T20:25:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/8ea100bbd72b467e8ad0a4377d7c5bb5 is 50, key is test_row_0/B:col10/1731875147661/Put/seqid=0 2024-11-17T20:25:48,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742344_1520 (size=12301) 2024-11-17T20:25:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:48,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875209019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875209022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875209129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875209140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,290 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/afd6681a39414a2191392895b9cb7c50 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/afd6681a39414a2191392895b9cb7c50 2024-11-17T20:25:49,290 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/683550f821ac4c1a85b53a4271a8b827 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/683550f821ac4c1a85b53a4271a8b827 2024-11-17T20:25:49,293 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into afd6681a39414a2191392895b9cb7c50(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:49,293 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:49,293 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875148871; duration=0sec 2024-11-17T20:25:49,293 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:49,293 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:49,293 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:49,293 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 683550f821ac4c1a85b53a4271a8b827(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:49,293 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:49,293 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875148871; duration=0sec 2024-11-17T20:25:49,294 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:49,294 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:49,294 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:49,294 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:49,294 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:49,294 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b9f812895cfa4a6cb374be6cef66d79a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7f490fd05a2c45f7b331dc755398a40b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/1b39da7844b849c8a1c0d10ee8418a93] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.3 K 2024-11-17T20:25:49,294 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting b9f812895cfa4a6cb374be6cef66d79a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731875145557 2024-11-17T20:25:49,295 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f490fd05a2c45f7b331dc755398a40b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731875145664 2024-11-17T20:25:49,295 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b39da7844b849c8a1c0d10ee8418a93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731875146771 2024-11-17T20:25:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T20:25:49,301 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:49,301 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5d3e84175a4a43018fbec66d63a30ffd is 50, key is test_row_0/C:col10/1731875146771/Put/seqid=0 2024-11-17T20:25:49,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742345_1521 (size=12949) 2024-11-17T20:25:49,340 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/8ea100bbd72b467e8ad0a4377d7c5bb5 2024-11-17T20:25:49,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875209339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/8c1bf6bc20704706af4789db776f8974 is 50, key is test_row_0/C:col10/1731875147661/Put/seqid=0 2024-11-17T20:25:49,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742346_1522 (size=12301) 2024-11-17T20:25:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875209350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875209649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875209659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,711 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5d3e84175a4a43018fbec66d63a30ffd as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5d3e84175a4a43018fbec66d63a30ffd 2024-11-17T20:25:49,714 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 5d3e84175a4a43018fbec66d63a30ffd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:49,714 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:49,714 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875148872; duration=0sec 2024-11-17T20:25:49,714 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:49,714 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:49,750 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/8c1bf6bc20704706af4789db776f8974 2024-11-17T20:25:49,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c1b09f49311f4f679bb33da69ba0d194 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c1b09f49311f4f679bb33da69ba0d194 2024-11-17T20:25:49,756 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c1b09f49311f4f679bb33da69ba0d194, entries=150, sequenceid=289, filesize=12.0 K 2024-11-17T20:25:49,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/8ea100bbd72b467e8ad0a4377d7c5bb5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/8ea100bbd72b467e8ad0a4377d7c5bb5 2024-11-17T20:25:49,759 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/8ea100bbd72b467e8ad0a4377d7c5bb5, entries=150, sequenceid=289, filesize=12.0 K 2024-11-17T20:25:49,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/8c1bf6bc20704706af4789db776f8974 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/8c1bf6bc20704706af4789db776f8974 2024-11-17T20:25:49,763 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/8c1bf6bc20704706af4789db776f8974, entries=150, sequenceid=289, filesize=12.0 K 2024-11-17T20:25:49,764 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 6ca3813db0fd868a85c94bd0134efe1f in 839ms, sequenceid=289, compaction requested=false 2024-11-17T20:25:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-17T20:25:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-17T20:25:49,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-17T20:25:49,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6030 sec 2024-11-17T20:25:49,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.6070 sec 2024-11-17T20:25:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:49,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:49,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-17T20:25:49,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/8dab2cf0e94c4d80b10e60925187a68b is 50, key is test_row_0/A:col10/1731875149823/Put/seqid=0 2024-11-17T20:25:49,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742347_1523 (size=14741) 2024-11-17T20:25:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875209840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875209842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875209949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875209960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:50,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875210159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875210159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:50,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875210170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:50,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875210170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:50,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/8dab2cf0e94c4d80b10e60925187a68b 2024-11-17T20:25:50,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ab2e45030a624b669792f5f3842e8a84 is 50, key is test_row_0/B:col10/1731875149823/Put/seqid=0 2024-11-17T20:25:50,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742348_1524 (size=12301) 2024-11-17T20:25:50,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ab2e45030a624b669792f5f3842e8a84 2024-11-17T20:25:50,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/3266ef8e73de472fa16008d1449e7c8c is 50, key is test_row_0/C:col10/1731875149823/Put/seqid=0 2024-11-17T20:25:50,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742349_1525 (size=12301) 2024-11-17T20:25:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T20:25:50,309 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:50,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:50,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-17T20:25:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T20:25:50,311 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:50,311 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:50,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T20:25:50,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-17T20:25:50,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:50,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:50,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:50,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:50,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875210469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:50,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875210479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:50,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-17T20:25:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:50,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T20:25:50,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/3266ef8e73de472fa16008d1449e7c8c 2024-11-17T20:25:50,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/8dab2cf0e94c4d80b10e60925187a68b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/8dab2cf0e94c4d80b10e60925187a68b 2024-11-17T20:25:50,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/8dab2cf0e94c4d80b10e60925187a68b, entries=200, sequenceid=320, filesize=14.4 K 2024-11-17T20:25:50,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ab2e45030a624b669792f5f3842e8a84 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab2e45030a624b669792f5f3842e8a84 2024-11-17T20:25:50,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab2e45030a624b669792f5f3842e8a84, entries=150, sequenceid=320, filesize=12.0 K 2024-11-17T20:25:50,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/3266ef8e73de472fa16008d1449e7c8c as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3266ef8e73de472fa16008d1449e7c8c 2024-11-17T20:25:50,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3266ef8e73de472fa16008d1449e7c8c, entries=150, sequenceid=320, filesize=12.0 K 2024-11-17T20:25:50,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 6ca3813db0fd868a85c94bd0134efe1f in 841ms, sequenceid=320, compaction requested=true 2024-11-17T20:25:50,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:50,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:50,665 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:50,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:50,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:50,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:50,666 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:50,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:50,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:50,666 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:50,666 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:50,666 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:50,666 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/683550f821ac4c1a85b53a4271a8b827, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c1b09f49311f4f679bb33da69ba0d194, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/8dab2cf0e94c4d80b10e60925187a68b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=39.1 K 2024-11-17T20:25:50,667 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:50,667 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 683550f821ac4c1a85b53a4271a8b827, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731875146771 2024-11-17T20:25:50,667 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:50,667 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:50,667 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/afd6681a39414a2191392895b9cb7c50, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/8ea100bbd72b467e8ad0a4377d7c5bb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab2e45030a624b669792f5f3842e8a84] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.7 K 2024-11-17T20:25:50,667 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b09f49311f4f679bb33da69ba0d194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731875147636 2024-11-17T20:25:50,667 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting afd6681a39414a2191392895b9cb7c50, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731875146771 2024-11-17T20:25:50,667 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dab2cf0e94c4d80b10e60925187a68b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731875149007 2024-11-17T20:25:50,668 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea100bbd72b467e8ad0a4377d7c5bb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731875147636 2024-11-17T20:25:50,668 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ab2e45030a624b669792f5f3842e8a84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731875149007 2024-11-17T20:25:50,675 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:50,675 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:50,676 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/1c680ea0d7f24eccbd038a2f5254fd1b is 50, key is test_row_0/A:col10/1731875149823/Put/seqid=0 2024-11-17T20:25:50,676 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/717f1b8df19648f0a0706fba45318314 is 50, key is test_row_0/B:col10/1731875149823/Put/seqid=0 2024-11-17T20:25:50,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742350_1526 (size=13051) 2024-11-17T20:25:50,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742351_1527 (size=13051) 2024-11-17T20:25:50,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-17T20:25:50,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:50,769 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-17T20:25:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/22ae040c45da41f4a8562f1026e48ddb is 50, key is test_row_0/A:col10/1731875149839/Put/seqid=0 2024-11-17T20:25:50,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742352_1528 (size=12301) 2024-11-17T20:25:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T20:25:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:51,091 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/717f1b8df19648f0a0706fba45318314 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/717f1b8df19648f0a0706fba45318314 2024-11-17T20:25:51,091 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/1c680ea0d7f24eccbd038a2f5254fd1b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/1c680ea0d7f24eccbd038a2f5254fd1b 2024-11-17T20:25:51,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875211081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875211086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,096 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 1c680ea0d7f24eccbd038a2f5254fd1b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:51,096 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 717f1b8df19648f0a0706fba45318314(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:51,096 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875150665; duration=0sec 2024-11-17T20:25:51,096 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875150666; duration=0sec 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:51,096 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:51,096 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:51,097 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5d3e84175a4a43018fbec66d63a30ffd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/8c1bf6bc20704706af4789db776f8974, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3266ef8e73de472fa16008d1449e7c8c] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.7 K 2024-11-17T20:25:51,097 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3e84175a4a43018fbec66d63a30ffd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731875146771 2024-11-17T20:25:51,097 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1bf6bc20704706af4789db776f8974, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731875147636 2024-11-17T20:25:51,097 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3266ef8e73de472fa16008d1449e7c8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731875149007 2024-11-17T20:25:51,101 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#448 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:51,102 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/e6c457e5420e4fd190259f7f6b433c24 is 50, key is test_row_0/C:col10/1731875149823/Put/seqid=0 2024-11-17T20:25:51,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742353_1529 (size=13051) 2024-11-17T20:25:51,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875211170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,176 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/22ae040c45da41f4a8562f1026e48ddb 2024-11-17T20:25:51,176 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/14e0149f912f480086d62784ceb2259a is 50, key is test_row_0/B:col10/1731875149839/Put/seqid=0 2024-11-17T20:25:51,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742354_1530 (size=12301) 2024-11-17T20:25:51,184 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/14e0149f912f480086d62784ceb2259a 2024-11-17T20:25:51,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/16fbc3536fc94494a2d0cfc02b702cc0 is 50, key is test_row_0/C:col10/1731875149839/Put/seqid=0 2024-11-17T20:25:51,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742355_1531 (size=12301) 2024-11-17T20:25:51,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875211189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,197 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875211199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875211199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875211409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875211409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T20:25:51,508 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/e6c457e5420e4fd190259f7f6b433c24 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e6c457e5420e4fd190259f7f6b433c24 2024-11-17T20:25:51,512 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into e6c457e5420e4fd190259f7f6b433c24(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:51,512 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:51,512 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875150666; duration=0sec 2024-11-17T20:25:51,512 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:51,512 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:51,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9e06ff800cb94f18b1ed4615da1e0463, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/f830d8a272c54304a0e07959e228fad3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5bc1ae11e1bd4bb0a97750a0ad555a8f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/206c1ad9ee0240b3a727d6a2b29b054d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4cdaea5bfa6c43bbb2aac0c57d216951, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/19b9769e59ab4c48aab5b405ab436a26, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/142a2ca1a7de46f2900455688f68f8e0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4794e75b9e6e4815bb960de54ebb45a4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/6f2162f8b4034695881f99c080e7c657, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/a3e137c754b94a4083025a4867eeb17d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/d556ad8f9e5f483e94425a60af0a7e90, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/212c21e3ba3d4f1c994e597da1df8be0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/cf8ad4d2342f434aa9b3a439c3d3a2ec, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c17494f2a48d4d1085f3dfba4570c418, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5556d9e4ee8444449095b7bf2b3f4993, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ae26269aeef45059f9cfba04249f2c5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/485182ff03934a32934e001cd0e8a8ec, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0ae6db38f079437ca2aae1c2515704b0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3c096b1e0c5242a7b8a89b539a21c889, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/683550f821ac4c1a85b53a4271a8b827, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c1b09f49311f4f679bb33da69ba0d194, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/8dab2cf0e94c4d80b10e60925187a68b] to archive 2024-11-17T20:25:51,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:51,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9e06ff800cb94f18b1ed4615da1e0463 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9e06ff800cb94f18b1ed4615da1e0463 2024-11-17T20:25:51,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/f830d8a272c54304a0e07959e228fad3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/f830d8a272c54304a0e07959e228fad3 2024-11-17T20:25:51,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5bc1ae11e1bd4bb0a97750a0ad555a8f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5bc1ae11e1bd4bb0a97750a0ad555a8f 2024-11-17T20:25:51,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/206c1ad9ee0240b3a727d6a2b29b054d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/206c1ad9ee0240b3a727d6a2b29b054d 2024-11-17T20:25:51,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4cdaea5bfa6c43bbb2aac0c57d216951 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4cdaea5bfa6c43bbb2aac0c57d216951 2024-11-17T20:25:51,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/19b9769e59ab4c48aab5b405ab436a26 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/19b9769e59ab4c48aab5b405ab436a26 2024-11-17T20:25:51,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/142a2ca1a7de46f2900455688f68f8e0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/142a2ca1a7de46f2900455688f68f8e0 2024-11-17T20:25:51,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4794e75b9e6e4815bb960de54ebb45a4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/4794e75b9e6e4815bb960de54ebb45a4 2024-11-17T20:25:51,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/6f2162f8b4034695881f99c080e7c657 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/6f2162f8b4034695881f99c080e7c657 2024-11-17T20:25:51,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/a3e137c754b94a4083025a4867eeb17d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/a3e137c754b94a4083025a4867eeb17d 2024-11-17T20:25:51,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/d556ad8f9e5f483e94425a60af0a7e90 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/d556ad8f9e5f483e94425a60af0a7e90 2024-11-17T20:25:51,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/212c21e3ba3d4f1c994e597da1df8be0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/212c21e3ba3d4f1c994e597da1df8be0 2024-11-17T20:25:51,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/cf8ad4d2342f434aa9b3a439c3d3a2ec to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/cf8ad4d2342f434aa9b3a439c3d3a2ec 2024-11-17T20:25:51,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c17494f2a48d4d1085f3dfba4570c418 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c17494f2a48d4d1085f3dfba4570c418 2024-11-17T20:25:51,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5556d9e4ee8444449095b7bf2b3f4993 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/5556d9e4ee8444449095b7bf2b3f4993 2024-11-17T20:25:51,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ae26269aeef45059f9cfba04249f2c5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ae26269aeef45059f9cfba04249f2c5 2024-11-17T20:25:51,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/485182ff03934a32934e001cd0e8a8ec to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/485182ff03934a32934e001cd0e8a8ec 2024-11-17T20:25:51,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0ae6db38f079437ca2aae1c2515704b0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0ae6db38f079437ca2aae1c2515704b0 2024-11-17T20:25:51,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3c096b1e0c5242a7b8a89b539a21c889 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3c096b1e0c5242a7b8a89b539a21c889 2024-11-17T20:25:51,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/683550f821ac4c1a85b53a4271a8b827 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/683550f821ac4c1a85b53a4271a8b827 2024-11-17T20:25:51,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c1b09f49311f4f679bb33da69ba0d194 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c1b09f49311f4f679bb33da69ba0d194 2024-11-17T20:25:51,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/8dab2cf0e94c4d80b10e60925187a68b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/8dab2cf0e94c4d80b10e60925187a68b 2024-11-17T20:25:51,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ef6d866e4d83432eaa2e40133c8d1a2e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/890267f88bc1475994794ce89a6dc508, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4f01397ebdec48799309fb21932c4b4c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/95e8257e2dc948b583ecda6eab33179b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1857257196c74c7ea9dd7bbec0d9c191, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/326979d40e4d4ab487a74007c72a8544, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/09d475473f724e78bb2be56188ee6152, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/453738b006f44c5ea387ca22beaa83fe, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/838450d561e74ad4a5f785277e92374b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/c0a1925070864066a48454c848ed443c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/37f1e224eace492db7b146c0317438c2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/338fb57e91674a54b5c23fe9f24c4119, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1162cd367f514375a93d008102feac53, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/dbb2811229b64094a01a7e9fbcf29b39, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3163b3463e3e4544a287da8b1979ebb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4fdf6683e0dd494a86f3d48a2b31870c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/aa00dc5efe894b489af8cec2d630b88b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3f326d7a7db84b589721d7d8e6b70149, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/afd6681a39414a2191392895b9cb7c50, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/16fba09dacaf4dd796cbb0d07379ed77, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/8ea100bbd72b467e8ad0a4377d7c5bb5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab2e45030a624b669792f5f3842e8a84] to archive 2024-11-17T20:25:51,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:51,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ef6d866e4d83432eaa2e40133c8d1a2e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ef6d866e4d83432eaa2e40133c8d1a2e 2024-11-17T20:25:51,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/890267f88bc1475994794ce89a6dc508 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/890267f88bc1475994794ce89a6dc508 2024-11-17T20:25:51,560 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4f01397ebdec48799309fb21932c4b4c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4f01397ebdec48799309fb21932c4b4c 2024-11-17T20:25:51,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/95e8257e2dc948b583ecda6eab33179b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/95e8257e2dc948b583ecda6eab33179b 2024-11-17T20:25:51,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1857257196c74c7ea9dd7bbec0d9c191 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1857257196c74c7ea9dd7bbec0d9c191 2024-11-17T20:25:51,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/326979d40e4d4ab487a74007c72a8544 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/326979d40e4d4ab487a74007c72a8544 2024-11-17T20:25:51,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/09d475473f724e78bb2be56188ee6152 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/09d475473f724e78bb2be56188ee6152 2024-11-17T20:25:51,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/453738b006f44c5ea387ca22beaa83fe to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/453738b006f44c5ea387ca22beaa83fe 2024-11-17T20:25:51,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/838450d561e74ad4a5f785277e92374b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/838450d561e74ad4a5f785277e92374b 2024-11-17T20:25:51,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/c0a1925070864066a48454c848ed443c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/c0a1925070864066a48454c848ed443c 2024-11-17T20:25:51,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/37f1e224eace492db7b146c0317438c2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/37f1e224eace492db7b146c0317438c2 2024-11-17T20:25:51,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/338fb57e91674a54b5c23fe9f24c4119 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/338fb57e91674a54b5c23fe9f24c4119 2024-11-17T20:25:51,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1162cd367f514375a93d008102feac53 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1162cd367f514375a93d008102feac53 2024-11-17T20:25:51,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/dbb2811229b64094a01a7e9fbcf29b39 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/dbb2811229b64094a01a7e9fbcf29b39 2024-11-17T20:25:51,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3163b3463e3e4544a287da8b1979ebb5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3163b3463e3e4544a287da8b1979ebb5 2024-11-17T20:25:51,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4fdf6683e0dd494a86f3d48a2b31870c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/4fdf6683e0dd494a86f3d48a2b31870c 2024-11-17T20:25:51,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/aa00dc5efe894b489af8cec2d630b88b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/aa00dc5efe894b489af8cec2d630b88b 2024-11-17T20:25:51,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3f326d7a7db84b589721d7d8e6b70149 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/3f326d7a7db84b589721d7d8e6b70149 2024-11-17T20:25:51,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/afd6681a39414a2191392895b9cb7c50 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/afd6681a39414a2191392895b9cb7c50 2024-11-17T20:25:51,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/16fba09dacaf4dd796cbb0d07379ed77 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/16fba09dacaf4dd796cbb0d07379ed77 2024-11-17T20:25:51,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/8ea100bbd72b467e8ad0a4377d7c5bb5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/8ea100bbd72b467e8ad0a4377d7c5bb5 2024-11-17T20:25:51,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab2e45030a624b669792f5f3842e8a84 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab2e45030a624b669792f5f3842e8a84 2024-11-17T20:25:51,584 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/86bd1a3b326a481c9d86d73dced584d1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/215e7166512c4d79bf1c7ae1d56d0024, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9425beb3893f4933802d65e24ba8c8d3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/a137f1e4425d497a959475dd1992e399, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b844f415b234f1e81c2ac159eb45f29, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/54076310ebfc432d927ad5c48d1d741a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/4af7850b8fc34ca5ab59d030056ffe0e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d850612078604e8784142d1298542d25, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/83acdc5a1f3343d1899cdb565ff74c2c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/864b8f895d31431996afa05655c5de86, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3abd1431489b4b71982bc832db92a751, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5438bd46ca77434d8b9d506708712dd2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5a50285588c542319438023f56e9483d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/23daf64a98394f0f8dcee24efbdc504e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/bf2219fb3c9546738d88efed734f0575, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b9f812895cfa4a6cb374be6cef66d79a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/2efefc6f15944877be05091c7a9d1f8c, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7f490fd05a2c45f7b331dc755398a40b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5d3e84175a4a43018fbec66d63a30ffd, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/1b39da7844b849c8a1c0d10ee8418a93, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/8c1bf6bc20704706af4789db776f8974, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3266ef8e73de472fa16008d1449e7c8c] to archive 2024-11-17T20:25:51,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:25:51,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/86bd1a3b326a481c9d86d73dced584d1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/86bd1a3b326a481c9d86d73dced584d1 2024-11-17T20:25:51,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/215e7166512c4d79bf1c7ae1d56d0024 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/215e7166512c4d79bf1c7ae1d56d0024 2024-11-17T20:25:51,587 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9425beb3893f4933802d65e24ba8c8d3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9425beb3893f4933802d65e24ba8c8d3 2024-11-17T20:25:51,588 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/a137f1e4425d497a959475dd1992e399 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/a137f1e4425d497a959475dd1992e399 2024-11-17T20:25:51,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b844f415b234f1e81c2ac159eb45f29 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b844f415b234f1e81c2ac159eb45f29 2024-11-17T20:25:51,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/54076310ebfc432d927ad5c48d1d741a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/54076310ebfc432d927ad5c48d1d741a 2024-11-17T20:25:51,590 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/4af7850b8fc34ca5ab59d030056ffe0e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/4af7850b8fc34ca5ab59d030056ffe0e 2024-11-17T20:25:51,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d850612078604e8784142d1298542d25 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d850612078604e8784142d1298542d25 2024-11-17T20:25:51,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/83acdc5a1f3343d1899cdb565ff74c2c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/83acdc5a1f3343d1899cdb565ff74c2c 2024-11-17T20:25:51,592 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/16fbc3536fc94494a2d0cfc02b702cc0 2024-11-17T20:25:51,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/864b8f895d31431996afa05655c5de86 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/864b8f895d31431996afa05655c5de86 2024-11-17T20:25:51,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3abd1431489b4b71982bc832db92a751 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3abd1431489b4b71982bc832db92a751 2024-11-17T20:25:51,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/22ae040c45da41f4a8562f1026e48ddb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/22ae040c45da41f4a8562f1026e48ddb 2024-11-17T20:25:51,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5438bd46ca77434d8b9d506708712dd2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5438bd46ca77434d8b9d506708712dd2 2024-11-17T20:25:51,598 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5a50285588c542319438023f56e9483d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5a50285588c542319438023f56e9483d 2024-11-17T20:25:51,599 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/22ae040c45da41f4a8562f1026e48ddb, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/14e0149f912f480086d62784ceb2259a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/14e0149f912f480086d62784ceb2259a 2024-11-17T20:25:51,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/23daf64a98394f0f8dcee24efbdc504e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/23daf64a98394f0f8dcee24efbdc504e 2024-11-17T20:25:51,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/bf2219fb3c9546738d88efed734f0575 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/bf2219fb3c9546738d88efed734f0575 2024-11-17T20:25:51,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b9f812895cfa4a6cb374be6cef66d79a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b9f812895cfa4a6cb374be6cef66d79a 2024-11-17T20:25:51,603 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/14e0149f912f480086d62784ceb2259a, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:51,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/16fbc3536fc94494a2d0cfc02b702cc0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/16fbc3536fc94494a2d0cfc02b702cc0 2024-11-17T20:25:51,605 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/2efefc6f15944877be05091c7a9d1f8c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/2efefc6f15944877be05091c7a9d1f8c 2024-11-17T20:25:51,606 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7f490fd05a2c45f7b331dc755398a40b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7f490fd05a2c45f7b331dc755398a40b 2024-11-17T20:25:51,607 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5d3e84175a4a43018fbec66d63a30ffd to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5d3e84175a4a43018fbec66d63a30ffd 2024-11-17T20:25:51,607 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/16fbc3536fc94494a2d0cfc02b702cc0, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T20:25:51,608 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/1b39da7844b849c8a1c0d10ee8418a93 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/1b39da7844b849c8a1c0d10ee8418a93 2024-11-17T20:25:51,608 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 6ca3813db0fd868a85c94bd0134efe1f in 840ms, sequenceid=328, compaction requested=false 2024-11-17T20:25:51,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:51,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:51,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-17T20:25:51,609 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/8c1bf6bc20704706af4789db776f8974 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/8c1bf6bc20704706af4789db776f8974 2024-11-17T20:25:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-17T20:25:51,610 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b0b0ae8de03:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3266ef8e73de472fa16008d1449e7c8c to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/3266ef8e73de472fa16008d1449e7c8c 2024-11-17T20:25:51,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-17T20:25:51,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2980 sec 2024-11-17T20:25:51,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.3030 sec 2024-11-17T20:25:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:51,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:51,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-17T20:25:51,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/ffd10ced321946c2b1fca1ca3d593938 is 50, key is test_row_0/A:col10/1731875151080/Put/seqid=0 2024-11-17T20:25:51,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742356_1532 (size=14741) 2024-11-17T20:25:51,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875211728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875211729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875211729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875211840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875211840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875212049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:52,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:52,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875212049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/ffd10ced321946c2b1fca1ca3d593938 2024-11-17T20:25:52,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/08dcb97137a44d4daa93a6b98d800e32 is 50, key is test_row_0/B:col10/1731875151080/Put/seqid=0 2024-11-17T20:25:52,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742357_1533 (size=12301) 2024-11-17T20:25:52,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875212249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:52,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875212359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:52,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875212359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T20:25:52,459 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:52,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-17T20:25:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T20:25:52,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:52,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:52,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:52,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/08dcb97137a44d4daa93a6b98d800e32 2024-11-17T20:25:52,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/d9bb8ab210d94f7381b144031b0701b3 is 50, key is test_row_0/C:col10/1731875151080/Put/seqid=0 2024-11-17T20:25:52,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742358_1534 (size=12301) 2024-11-17T20:25:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T20:25:52,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-17T20:25:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:52,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T20:25:52,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-17T20:25:52,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:52,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:52,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875212869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875212869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:52,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/d9bb8ab210d94f7381b144031b0701b3 2024-11-17T20:25:52,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/ffd10ced321946c2b1fca1ca3d593938 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ffd10ced321946c2b1fca1ca3d593938 2024-11-17T20:25:52,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ffd10ced321946c2b1fca1ca3d593938, entries=200, sequenceid=360, filesize=14.4 K 2024-11-17T20:25:52,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/08dcb97137a44d4daa93a6b98d800e32 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/08dcb97137a44d4daa93a6b98d800e32 2024-11-17T20:25:52,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/08dcb97137a44d4daa93a6b98d800e32, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T20:25:52,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/d9bb8ab210d94f7381b144031b0701b3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d9bb8ab210d94f7381b144031b0701b3 2024-11-17T20:25:52,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d9bb8ab210d94f7381b144031b0701b3, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T20:25:52,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 6ca3813db0fd868a85c94bd0134efe1f in 1244ms, sequenceid=360, compaction requested=true 2024-11-17T20:25:52,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:52,959 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:52,959 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:52,962 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:52,962 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:52,962 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,962 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/1c680ea0d7f24eccbd038a2f5254fd1b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/22ae040c45da41f4a8562f1026e48ddb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ffd10ced321946c2b1fca1ca3d593938] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=39.2 K 2024-11-17T20:25:52,962 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:52,962 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:52,962 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,963 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/717f1b8df19648f0a0706fba45318314, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/14e0149f912f480086d62784ceb2259a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/08dcb97137a44d4daa93a6b98d800e32] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.8 K 2024-11-17T20:25:52,963 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c680ea0d7f24eccbd038a2f5254fd1b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731875149007 2024-11-17T20:25:52,963 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ae040c45da41f4a8562f1026e48ddb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731875149833 2024-11-17T20:25:52,963 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 717f1b8df19648f0a0706fba45318314, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731875149007 2024-11-17T20:25:52,963 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd10ced321946c2b1fca1ca3d593938, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731875151077 2024-11-17T20:25:52,963 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e0149f912f480086d62784ceb2259a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731875149833 2024-11-17T20:25:52,963 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 08dcb97137a44d4daa93a6b98d800e32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731875151077 2024-11-17T20:25:52,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-17T20:25:52,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,968 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-17T20:25:52,969 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:52,970 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:52,970 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/797ac92551ca4eec942ace4fc66a06f1 is 50, key is test_row_0/B:col10/1731875151080/Put/seqid=0 2024-11-17T20:25:52,970 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/e14fba5ce6c64f21893f6dc19fbae65e is 50, key is test_row_0/A:col10/1731875151080/Put/seqid=0 2024-11-17T20:25:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3d968aa9736a4030a00baa200f44542e is 50, key is test_row_0/A:col10/1731875151723/Put/seqid=0 2024-11-17T20:25:52,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742360_1536 (size=12404) 2024-11-17T20:25:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742359_1535 (size=12404) 2024-11-17T20:25:52,983 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/797ac92551ca4eec942ace4fc66a06f1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/797ac92551ca4eec942ace4fc66a06f1 2024-11-17T20:25:52,989 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 797ac92551ca4eec942ace4fc66a06f1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:52,989 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:52,989 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875152959; duration=0sec 2024-11-17T20:25:52,989 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:52,989 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:52,989 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:52,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742361_1537 (size=9857) 2024-11-17T20:25:52,993 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:52,993 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:52,994 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:52,994 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3d968aa9736a4030a00baa200f44542e 2024-11-17T20:25:52,994 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e6c457e5420e4fd190259f7f6b433c24, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/16fbc3536fc94494a2d0cfc02b702cc0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d9bb8ab210d94f7381b144031b0701b3] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.8 K 2024-11-17T20:25:52,994 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c457e5420e4fd190259f7f6b433c24, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731875149007 2024-11-17T20:25:52,994 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 16fbc3536fc94494a2d0cfc02b702cc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731875149833 2024-11-17T20:25:52,995 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting d9bb8ab210d94f7381b144031b0701b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731875151077 2024-11-17T20:25:53,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/170a58747d10496cb4c9f8dda3c6440e is 50, key is test_row_0/B:col10/1731875151723/Put/seqid=0 2024-11-17T20:25:53,015 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:53,015 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/068ec0deca8048fa960b189ea82e6cde is 50, key is test_row_0/C:col10/1731875151080/Put/seqid=0 2024-11-17T20:25:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742362_1538 (size=9857) 2024-11-17T20:25:53,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742363_1539 (size=12404) 2024-11-17T20:25:53,042 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/068ec0deca8048fa960b189ea82e6cde as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/068ec0deca8048fa960b189ea82e6cde 2024-11-17T20:25:53,050 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 068ec0deca8048fa960b189ea82e6cde(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:53,050 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:53,050 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875152959; duration=0sec 2024-11-17T20:25:53,050 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:53,050 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T20:25:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:53,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:53,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875213259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875213259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875213259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,267 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875213369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875213369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,381 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/e14fba5ce6c64f21893f6dc19fbae65e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/e14fba5ce6c64f21893f6dc19fbae65e 2024-11-17T20:25:53,384 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into e14fba5ce6c64f21893f6dc19fbae65e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:53,384 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:53,384 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875152959; duration=0sec 2024-11-17T20:25:53,384 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:53,384 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:53,437 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/170a58747d10496cb4c9f8dda3c6440e 2024-11-17T20:25:53,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/e44e02c14c5240bca610804c00043c6f is 50, key is test_row_0/C:col10/1731875151723/Put/seqid=0 2024-11-17T20:25:53,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742364_1540 (size=9857) 2024-11-17T20:25:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875213579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875213579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T20:25:53,846 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/e44e02c14c5240bca610804c00043c6f 2024-11-17T20:25:53,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3d968aa9736a4030a00baa200f44542e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3d968aa9736a4030a00baa200f44542e 2024-11-17T20:25:53,852 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3d968aa9736a4030a00baa200f44542e, entries=100, sequenceid=367, filesize=9.6 K 2024-11-17T20:25:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/170a58747d10496cb4c9f8dda3c6440e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/170a58747d10496cb4c9f8dda3c6440e 2024-11-17T20:25:53,855 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/170a58747d10496cb4c9f8dda3c6440e, entries=100, sequenceid=367, filesize=9.6 K 2024-11-17T20:25:53,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/e44e02c14c5240bca610804c00043c6f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e44e02c14c5240bca610804c00043c6f 2024-11-17T20:25:53,858 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e44e02c14c5240bca610804c00043c6f, entries=100, sequenceid=367, filesize=9.6 K 2024-11-17T20:25:53,858 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 6ca3813db0fd868a85c94bd0134efe1f in 890ms, sequenceid=367, compaction requested=false 2024-11-17T20:25:53,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:53,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:53,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-17T20:25:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-17T20:25:53,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T20:25:53,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3500 sec 2024-11-17T20:25:53,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.4020 sec 2024-11-17T20:25:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:53,883 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:53,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-17T20:25:53,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/7ed5ee474a264459b2254d0e6ee0613f is 50, key is test_row_0/A:col10/1731875153257/Put/seqid=0 2024-11-17T20:25:53,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742365_1541 (size=14741) 2024-11-17T20:25:53,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875213887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875213889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875213889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875213891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875213999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875213999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875214210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875214210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/7ed5ee474a264459b2254d0e6ee0613f 2024-11-17T20:25:54,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/5dadd261151e435e8592d6956a1925ca is 50, key is test_row_0/B:col10/1731875153257/Put/seqid=0 2024-11-17T20:25:54,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742366_1542 (size=12301) 2024-11-17T20:25:54,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875214399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875214399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875214519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875214519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T20:25:54,610 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:54,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-17T20:25:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T20:25:54,612 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:54,612 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:54,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:54,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/5dadd261151e435e8592d6956a1925ca 2024-11-17T20:25:54,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/757bb5527fdf468ba8288d0de712724f is 50, key is test_row_0/C:col10/1731875153257/Put/seqid=0 2024-11-17T20:25:54,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742367_1543 (size=12301) 2024-11-17T20:25:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T20:25:54,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-17T20:25:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:54,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:54,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-17T20:25:54,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:54,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:54,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:54,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:54,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T20:25:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875215029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875215030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:55,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-17T20:25:55,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:55,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:55,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:55,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:55,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:55,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/757bb5527fdf468ba8288d0de712724f 2024-11-17T20:25:55,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/7ed5ee474a264459b2254d0e6ee0613f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ed5ee474a264459b2254d0e6ee0613f 2024-11-17T20:25:55,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ed5ee474a264459b2254d0e6ee0613f, entries=200, sequenceid=400, filesize=14.4 K 2024-11-17T20:25:55,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/5dadd261151e435e8592d6956a1925ca as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/5dadd261151e435e8592d6956a1925ca 2024-11-17T20:25:55,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/5dadd261151e435e8592d6956a1925ca, entries=150, sequenceid=400, filesize=12.0 K 2024-11-17T20:25:55,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/757bb5527fdf468ba8288d0de712724f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/757bb5527fdf468ba8288d0de712724f 2024-11-17T20:25:55,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/757bb5527fdf468ba8288d0de712724f, entries=150, sequenceid=400, filesize=12.0 K 2024-11-17T20:25:55,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 6ca3813db0fd868a85c94bd0134efe1f in 1236ms, sequenceid=400, compaction requested=true 2024-11-17T20:25:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:55,119 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:55,119 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:55,120 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:55,120 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37002 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:55,120 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:55,120 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:55,120 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:55,120 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:55,120 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/797ac92551ca4eec942ace4fc66a06f1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/170a58747d10496cb4c9f8dda3c6440e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/5dadd261151e435e8592d6956a1925ca] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=33.8 K 2024-11-17T20:25:55,120 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/e14fba5ce6c64f21893f6dc19fbae65e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3d968aa9736a4030a00baa200f44542e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ed5ee474a264459b2254d0e6ee0613f] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.1 K 2024-11-17T20:25:55,121 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e14fba5ce6c64f21893f6dc19fbae65e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731875151077 2024-11-17T20:25:55,121 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 797ac92551ca4eec942ace4fc66a06f1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731875151077 2024-11-17T20:25:55,121 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 170a58747d10496cb4c9f8dda3c6440e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731875151723 2024-11-17T20:25:55,121 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d968aa9736a4030a00baa200f44542e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731875151723 2024-11-17T20:25:55,121 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dadd261151e435e8592d6956a1925ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731875153257 2024-11-17T20:25:55,121 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed5ee474a264459b2254d0e6ee0613f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731875153250 2024-11-17T20:25:55,127 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:55,127 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/35631cf2b8674ebba235e81a2f5ca7bf is 50, key is test_row_0/B:col10/1731875153257/Put/seqid=0 2024-11-17T20:25:55,128 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:55,128 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/9488b1e4d94d4f2d81069857937d8bb0 is 50, key is test_row_0/A:col10/1731875153257/Put/seqid=0 2024-11-17T20:25:55,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742368_1544 (size=12507) 2024-11-17T20:25:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742369_1545 (size=12507) 2024-11-17T20:25:55,135 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/35631cf2b8674ebba235e81a2f5ca7bf as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/35631cf2b8674ebba235e81a2f5ca7bf 2024-11-17T20:25:55,139 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 35631cf2b8674ebba235e81a2f5ca7bf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:55,139 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:55,139 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875155119; duration=0sec 2024-11-17T20:25:55,139 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:55,139 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:55,139 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:55,140 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:55,140 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:55,140 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:55,140 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/068ec0deca8048fa960b189ea82e6cde, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e44e02c14c5240bca610804c00043c6f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/757bb5527fdf468ba8288d0de712724f] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=33.8 K 2024-11-17T20:25:55,140 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 068ec0deca8048fa960b189ea82e6cde, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731875151077 2024-11-17T20:25:55,140 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting e44e02c14c5240bca610804c00043c6f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731875151723 2024-11-17T20:25:55,141 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 757bb5527fdf468ba8288d0de712724f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731875153257 2024-11-17T20:25:55,145 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:55,145 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/7fd31e8c137a49f8a05ba1b838098d51 is 50, key is test_row_0/C:col10/1731875153257/Put/seqid=0 2024-11-17T20:25:55,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742370_1546 (size=12507) 2024-11-17T20:25:55,153 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/7fd31e8c137a49f8a05ba1b838098d51 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7fd31e8c137a49f8a05ba1b838098d51 2024-11-17T20:25:55,156 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 7fd31e8c137a49f8a05ba1b838098d51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:55,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:55,156 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875155119; duration=0sec 2024-11-17T20:25:55,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:55,156 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:55,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-17T20:25:55,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:55,221 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-17T20:25:55,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3e34dcc62bc94694a61183bc48268df2 is 50, key is test_row_0/A:col10/1731875153883/Put/seqid=0 2024-11-17T20:25:55,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742371_1547 (size=9857) 2024-11-17T20:25:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T20:25:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:55,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:55,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875215395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875215409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:55,413 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875215419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:55,423 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875215499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:55,536 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/9488b1e4d94d4f2d81069857937d8bb0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9488b1e4d94d4f2d81069857937d8bb0 2024-11-17T20:25:55,540 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 9488b1e4d94d4f2d81069857937d8bb0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:55,540 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:55,540 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875155119; duration=0sec 2024-11-17T20:25:55,540 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:55,540 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:55,628 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3e34dcc62bc94694a61183bc48268df2 2024-11-17T20:25:55,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/03dcdc7c93de4f88bac04a67ee004926 is 50, key is test_row_0/B:col10/1731875153883/Put/seqid=0 2024-11-17T20:25:55,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742372_1548 (size=9857) 2024-11-17T20:25:55,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875215709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T20:25:56,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875216019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:56,036 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/03dcdc7c93de4f88bac04a67ee004926 2024-11-17T20:25:56,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5886a7afdf1f45329a1044a80becaf5b is 50, key is test_row_0/C:col10/1731875153883/Put/seqid=0 2024-11-17T20:25:56,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742373_1549 (size=9857) 2024-11-17T20:25:56,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875216049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:56,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875216049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:56,052 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,052 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:56,445 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5886a7afdf1f45329a1044a80becaf5b 2024-11-17T20:25:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/3e34dcc62bc94694a61183bc48268df2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3e34dcc62bc94694a61183bc48268df2 2024-11-17T20:25:56,450 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3e34dcc62bc94694a61183bc48268df2, entries=100, sequenceid=408, filesize=9.6 K 2024-11-17T20:25:56,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/03dcdc7c93de4f88bac04a67ee004926 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/03dcdc7c93de4f88bac04a67ee004926 2024-11-17T20:25:56,453 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/03dcdc7c93de4f88bac04a67ee004926, entries=100, sequenceid=408, filesize=9.6 K 2024-11-17T20:25:56,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/5886a7afdf1f45329a1044a80becaf5b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5886a7afdf1f45329a1044a80becaf5b 2024-11-17T20:25:56,456 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5886a7afdf1f45329a1044a80becaf5b, entries=100, sequenceid=408, filesize=9.6 K 2024-11-17T20:25:56,457 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 6ca3813db0fd868a85c94bd0134efe1f in 1236ms, sequenceid=408, compaction requested=false 2024-11-17T20:25:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-17T20:25:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-17T20:25:56,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-17T20:25:56,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8460 sec 2024-11-17T20:25:56,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.8500 sec 2024-11-17T20:25:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:56,534 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:56,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-17T20:25:56,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/bfd09e4783744fc2990cfcf159bef510 is 50, key is test_row_0/A:col10/1731875156533/Put/seqid=0 2024-11-17T20:25:56,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742374_1550 (size=14741) 2024-11-17T20:25:56,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875216541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:56,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875216650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T20:25:56,760 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:25:56,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:25:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-11-17T20:25:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-17T20:25:56,762 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:25:56,762 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:25:56,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:25:56,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875216859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-17T20:25:56,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:56,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:56,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/bfd09e4783744fc2990cfcf159bef510 2024-11-17T20:25:56,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1aea8e5932e9476183442bf4de5d5e14 is 50, key is test_row_0/B:col10/1731875156533/Put/seqid=0 2024-11-17T20:25:56,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742375_1551 (size=12301) 2024-11-17T20:25:57,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:57,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:57,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-17T20:25:57,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875217169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1aea8e5932e9476183442bf4de5d5e14 2024-11-17T20:25:57,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/b4319f80e2d34e4cb45e6d609db75026 is 50, key is test_row_0/C:col10/1731875156533/Put/seqid=0 2024-11-17T20:25:57,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742376_1552 (size=12301) 2024-11-17T20:25:57,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/b4319f80e2d34e4cb45e6d609db75026 2024-11-17T20:25:57,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/bfd09e4783744fc2990cfcf159bef510 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/bfd09e4783744fc2990cfcf159bef510 2024-11-17T20:25:57,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/bfd09e4783744fc2990cfcf159bef510, entries=200, sequenceid=440, filesize=14.4 K 2024-11-17T20:25:57,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/1aea8e5932e9476183442bf4de5d5e14 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1aea8e5932e9476183442bf4de5d5e14 2024-11-17T20:25:57,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1aea8e5932e9476183442bf4de5d5e14, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T20:25:57,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/b4319f80e2d34e4cb45e6d609db75026 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b4319f80e2d34e4cb45e6d609db75026 2024-11-17T20:25:57,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b4319f80e2d34e4cb45e6d609db75026, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T20:25:57,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 6ca3813db0fd868a85c94bd0134efe1f in 846ms, sequenceid=440, compaction requested=true 2024-11-17T20:25:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:25:57,380 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:25:57,380 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:25:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:57,381 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:57,381 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37105 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:57,381 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:25:57,381 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:25:57,381 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,381 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,381 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9488b1e4d94d4f2d81069857937d8bb0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3e34dcc62bc94694a61183bc48268df2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/bfd09e4783744fc2990cfcf159bef510] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=36.2 K 2024-11-17T20:25:57,381 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/35631cf2b8674ebba235e81a2f5ca7bf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/03dcdc7c93de4f88bac04a67ee004926, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1aea8e5932e9476183442bf4de5d5e14] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=33.9 K 2024-11-17T20:25:57,382 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9488b1e4d94d4f2d81069857937d8bb0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731875153257 2024-11-17T20:25:57,382 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 35631cf2b8674ebba235e81a2f5ca7bf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731875153257 2024-11-17T20:25:57,382 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 03dcdc7c93de4f88bac04a67ee004926, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731875153883 2024-11-17T20:25:57,382 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e34dcc62bc94694a61183bc48268df2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731875153883 2024-11-17T20:25:57,382 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aea8e5932e9476183442bf4de5d5e14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731875155364 2024-11-17T20:25:57,382 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfd09e4783744fc2990cfcf159bef510, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731875155364 2024-11-17T20:25:57,389 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-17T20:25:57,390 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/41fc269460ec4ba796a98113c9d81b1e is 50, key is test_row_0/B:col10/1731875156533/Put/seqid=0 2024-11-17T20:25:57,391 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:57,391 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/638097a3079f4074ad668153e35f2245 is 50, key is test_row_0/A:col10/1731875156533/Put/seqid=0 2024-11-17T20:25:57,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742377_1553 (size=12609) 2024-11-17T20:25:57,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742378_1554 (size=12609) 2024-11-17T20:25:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:57,457 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:57,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:25:57,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c0ecbdbd7fb5439dac290642a6b20d60 is 50, key is test_row_0/A:col10/1731875157448/Put/seqid=0 2024-11-17T20:25:57,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742379_1555 (size=17181) 2024-11-17T20:25:57,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c0ecbdbd7fb5439dac290642a6b20d60 2024-11-17T20:25:57,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/0b5a980811cc4270a3597e50e8b32802 is 50, key is test_row_0/B:col10/1731875157448/Put/seqid=0 2024-11-17T20:25:57,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742380_1556 (size=12301) 2024-11-17T20:25:57,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875217572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875217574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875217679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875217689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875217689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,801 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/41fc269460ec4ba796a98113c9d81b1e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/41fc269460ec4ba796a98113c9d81b1e 2024-11-17T20:25:57,802 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/638097a3079f4074ad668153e35f2245 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/638097a3079f4074ad668153e35f2245 2024-11-17T20:25:57,805 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into 41fc269460ec4ba796a98113c9d81b1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:57,805 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:57,805 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=13, startTime=1731875157380; duration=0sec 2024-11-17T20:25:57,805 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:25:57,805 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:25:57,805 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:25:57,806 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 638097a3079f4074ad668153e35f2245(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:57,806 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:57,806 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=13, startTime=1731875157380; duration=0sec 2024-11-17T20:25:57,806 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:57,806 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:25:57,807 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:25:57,807 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:25:57,807 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,807 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7fd31e8c137a49f8a05ba1b838098d51, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5886a7afdf1f45329a1044a80becaf5b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b4319f80e2d34e4cb45e6d609db75026] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=33.9 K 2024-11-17T20:25:57,807 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd31e8c137a49f8a05ba1b838098d51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731875153257 2024-11-17T20:25:57,807 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5886a7afdf1f45329a1044a80becaf5b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731875153883 2024-11-17T20:25:57,808 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting b4319f80e2d34e4cb45e6d609db75026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731875155364 2024-11-17T20:25:57,823 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:25:57,824 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/0b16b8ac51d84feb986343f1f6b2d127 is 50, key is test_row_0/C:col10/1731875156533/Put/seqid=0 2024-11-17T20:25:57,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742381_1557 (size=12609) 2024-11-17T20:25:57,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-17T20:25:57,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875217899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875217899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:57,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:57,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/0b5a980811cc4270a3597e50e8b32802 2024-11-17T20:25:57,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/91a0f2845c314c8c9fb11efe93a689bc is 50, key is test_row_0/C:col10/1731875157448/Put/seqid=0 2024-11-17T20:25:57,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742382_1558 (size=12301) 2024-11-17T20:25:57,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:57,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:57,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:58,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50166 deadline: 1731875218069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50220 deadline: 1731875218070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,075 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:58,076 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in 6ca3813db0fd868a85c94bd0134efe1f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:25:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:58,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:58,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:58,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:58,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:58,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:58,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875218210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875218210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,231 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/0b16b8ac51d84feb986343f1f6b2d127 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b16b8ac51d84feb986343f1f6b2d127 2024-11-17T20:25:58,234 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 0b16b8ac51d84feb986343f1f6b2d127(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:25:58,234 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:58,234 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=13, startTime=1731875157380; duration=0sec 2024-11-17T20:25:58,234 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:25:58,234 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:25:58,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:58,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:58,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:58,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:58,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:58,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:25:58,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/91a0f2845c314c8c9fb11efe93a689bc 2024-11-17T20:25:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/c0ecbdbd7fb5439dac290642a6b20d60 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c0ecbdbd7fb5439dac290642a6b20d60 2024-11-17T20:25:58,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c0ecbdbd7fb5439dac290642a6b20d60, entries=250, sequenceid=452, filesize=16.8 K 2024-11-17T20:25:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/0b5a980811cc4270a3597e50e8b32802 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/0b5a980811cc4270a3597e50e8b32802 2024-11-17T20:25:58,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/0b5a980811cc4270a3597e50e8b32802, entries=150, sequenceid=452, filesize=12.0 K 2024-11-17T20:25:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/91a0f2845c314c8c9fb11efe93a689bc as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/91a0f2845c314c8c9fb11efe93a689bc 2024-11-17T20:25:58,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/91a0f2845c314c8c9fb11efe93a689bc, entries=150, sequenceid=452, filesize=12.0 K 2024-11-17T20:25:58,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6ca3813db0fd868a85c94bd0134efe1f in 882ms, sequenceid=452, compaction requested=false 2024-11-17T20:25:58,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:58,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-17T20:25:58,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:58,439 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-17T20:25:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/956e2cc73d5a453eac2faaf8ac36e276 is 50, key is test_row_0/A:col10/1731875157571/Put/seqid=0 2024-11-17T20:25:58,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742383_1559 (size=12301) 2024-11-17T20:25:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:58,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. as already flushing 2024-11-17T20:25:58,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875218730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875218731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875218736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,846 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/956e2cc73d5a453eac2faaf8ac36e276 2024-11-17T20:25:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875218840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/91509d414a8e4f0bb85870934f05a673 is 50, key is test_row_0/B:col10/1731875157571/Put/seqid=0 2024-11-17T20:25:58,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742384_1560 (size=12301) 2024-11-17T20:25:58,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875218849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875218849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:58,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-17T20:25:59,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875219060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875219060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875219060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:59,138 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-17T20:25:59,138 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,138 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,138 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,140 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-17T20:25:59,140 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,140 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,140 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,142 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-11-17T20:25:59,142 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,142 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,142 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,145 INFO [Thread-2065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2065. 2024-11-17T20:25:59,146 DEBUG [Thread-2065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,146 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,146 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,148 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-17T20:25:59,148 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,148 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,148 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,255 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/91509d414a8e4f0bb85870934f05a673 2024-11-17T20:25:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/75f1c4d9d43b42c984321952ed77aa22 is 50, key is test_row_0/C:col10/1731875157571/Put/seqid=0 2024-11-17T20:25:59,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742385_1561 (size=12301) 2024-11-17T20:25:59,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50194 deadline: 1731875219380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:59,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:59,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:25:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50152 deadline: 1731875219380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50218 deadline: 1731875219380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:25:59,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 , the old value is region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:25:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=2 because the exception is null or not the one we care about 2024-11-17T20:25:59,670 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/75f1c4d9d43b42c984321952ed77aa22 2024-11-17T20:25:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/956e2cc73d5a453eac2faaf8ac36e276 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/956e2cc73d5a453eac2faaf8ac36e276 2024-11-17T20:25:59,684 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/956e2cc73d5a453eac2faaf8ac36e276, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T20:25:59,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/91509d414a8e4f0bb85870934f05a673 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/91509d414a8e4f0bb85870934f05a673 2024-11-17T20:25:59,688 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/91509d414a8e4f0bb85870934f05a673, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T20:25:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/75f1c4d9d43b42c984321952ed77aa22 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/75f1c4d9d43b42c984321952ed77aa22 2024-11-17T20:25:59,692 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/75f1c4d9d43b42c984321952ed77aa22, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T20:25:59,693 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6ca3813db0fd868a85c94bd0134efe1f in 1254ms, sequenceid=479, compaction requested=true 2024-11-17T20:25:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:25:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:25:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-11-17T20:25:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-11-17T20:25:59,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-17T20:25:59,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9320 sec 2024-11-17T20:25:59,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 2.9360 sec 2024-11-17T20:25:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:25:59,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:25:59,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-17T20:25:59,893 INFO [Thread-2052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2052. 2024-11-17T20:25:59,893 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-11-17T20:25:59,893 INFO [Thread-2056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2056. 2024-11-17T20:25:59,893 DEBUG [Thread-2052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,893 DEBUG [Thread-2056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,893 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:25:59,893 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,893 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,893 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,893 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,893 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,893 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:25:59,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:25:59,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/ee41afb7d7694cae98245af0a412859b is 50, key is test_row_0/A:col10/1731875158715/Put/seqid=0 2024-11-17T20:25:59,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742386_1562 (size=12301) 2024-11-17T20:26:00,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/ee41afb7d7694cae98245af0a412859b 2024-11-17T20:26:00,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ca2829c12d6f4a1a944f193031ec87f7 is 50, key is test_row_0/B:col10/1731875158715/Put/seqid=0 2024-11-17T20:26:00,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742387_1563 (size=12301) 2024-11-17T20:26:00,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ca2829c12d6f4a1a944f193031ec87f7 2024-11-17T20:26:00,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/62955b61babd48a0bfb85c7fad3b43c9 is 50, key is test_row_0/C:col10/1731875158715/Put/seqid=0 2024-11-17T20:26:00,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742388_1564 (size=12301) 2024-11-17T20:26:00,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/62955b61babd48a0bfb85c7fad3b43c9 2024-11-17T20:26:00,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/ee41afb7d7694cae98245af0a412859b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ee41afb7d7694cae98245af0a412859b 2024-11-17T20:26:00,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ee41afb7d7694cae98245af0a412859b, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T20:26:00,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ca2829c12d6f4a1a944f193031ec87f7 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ca2829c12d6f4a1a944f193031ec87f7 2024-11-17T20:26:00,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ca2829c12d6f4a1a944f193031ec87f7, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T20:26:00,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/62955b61babd48a0bfb85c7fad3b43c9 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/62955b61babd48a0bfb85c7fad3b43c9 2024-11-17T20:26:00,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/62955b61babd48a0bfb85c7fad3b43c9, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T20:26:00,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 6ca3813db0fd868a85c94bd0134efe1f in 868ms, sequenceid=494, compaction requested=true 2024-11-17T20:26:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:26:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:00,761 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ca3813db0fd868a85c94bd0134efe1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:00,761 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:00,762 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:00,762 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:00,762 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/B is initiating minor compaction (all files) 2024-11-17T20:26:00,762 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/A is initiating minor compaction (all files) 2024-11-17T20:26:00,762 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/B in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:26:00,762 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/A in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:26:00,762 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/41fc269460ec4ba796a98113c9d81b1e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/0b5a980811cc4270a3597e50e8b32802, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/91509d414a8e4f0bb85870934f05a673, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ca2829c12d6f4a1a944f193031ec87f7] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=48.4 K 2024-11-17T20:26:00,762 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/638097a3079f4074ad668153e35f2245, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c0ecbdbd7fb5439dac290642a6b20d60, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/956e2cc73d5a453eac2faaf8ac36e276, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ee41afb7d7694cae98245af0a412859b] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=53.1 K 2024-11-17T20:26:00,762 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 41fc269460ec4ba796a98113c9d81b1e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731875155364 2024-11-17T20:26:00,762 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638097a3079f4074ad668153e35f2245, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731875155364 2024-11-17T20:26:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5a980811cc4270a3597e50e8b32802, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731875157448 2024-11-17T20:26:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0ecbdbd7fb5439dac290642a6b20d60, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731875157448 2024-11-17T20:26:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 91509d414a8e4f0bb85870934f05a673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731875157563 2024-11-17T20:26:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 956e2cc73d5a453eac2faaf8ac36e276, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731875157563 2024-11-17T20:26:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2829c12d6f4a1a944f193031ec87f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731875158715 2024-11-17T20:26:00,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee41afb7d7694cae98245af0a412859b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731875158715 2024-11-17T20:26:00,771 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#B#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:00,771 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ab9bc70c53634710afe67a363ae32de2 is 50, key is test_row_0/B:col10/1731875158715/Put/seqid=0 2024-11-17T20:26:00,772 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#A#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:00,772 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/0a7f1216a01f40f98770d08713dba064 is 50, key is test_row_0/A:col10/1731875158715/Put/seqid=0 2024-11-17T20:26:00,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742389_1565 (size=12745) 2024-11-17T20:26:00,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742390_1566 (size=12745) 2024-11-17T20:26:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-17T20:26:00,951 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:01,185 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/ab9bc70c53634710afe67a363ae32de2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab9bc70c53634710afe67a363ae32de2 2024-11-17T20:26:01,185 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/0a7f1216a01f40f98770d08713dba064 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0a7f1216a01f40f98770d08713dba064 2024-11-17T20:26:01,189 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/A of 6ca3813db0fd868a85c94bd0134efe1f into 0a7f1216a01f40f98770d08713dba064(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:01,189 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/B of 6ca3813db0fd868a85c94bd0134efe1f into ab9bc70c53634710afe67a363ae32de2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:01,190 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:26:01,190 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:26:01,190 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/A, priority=12, startTime=1731875160761; duration=0sec 2024-11-17T20:26:01,190 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/B, priority=12, startTime=1731875160761; duration=0sec 2024-11-17T20:26:01,190 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:01,190 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:01,190 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:A 2024-11-17T20:26:01,190 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:B 2024-11-17T20:26:01,190 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:01,191 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:01,191 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): 6ca3813db0fd868a85c94bd0134efe1f/C is initiating minor compaction (all files) 2024-11-17T20:26:01,191 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ca3813db0fd868a85c94bd0134efe1f/C in TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:26:01,191 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b16b8ac51d84feb986343f1f6b2d127, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/91a0f2845c314c8c9fb11efe93a689bc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/75f1c4d9d43b42c984321952ed77aa22, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/62955b61babd48a0bfb85c7fad3b43c9] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp, totalSize=48.4 K 2024-11-17T20:26:01,191 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b16b8ac51d84feb986343f1f6b2d127, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731875155364 2024-11-17T20:26:01,192 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a0f2845c314c8c9fb11efe93a689bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731875157448 2024-11-17T20:26:01,192 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f1c4d9d43b42c984321952ed77aa22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731875157563 2024-11-17T20:26:01,192 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62955b61babd48a0bfb85c7fad3b43c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731875158715 2024-11-17T20:26:01,200 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ca3813db0fd868a85c94bd0134efe1f#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:01,200 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/0e406c2dec76403eb647711cd09f4848 is 50, key is test_row_0/C:col10/1731875158715/Put/seqid=0 2024-11-17T20:26:01,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742391_1567 (size=12745) 2024-11-17T20:26:01,611 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/0e406c2dec76403eb647711cd09f4848 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0e406c2dec76403eb647711cd09f4848 2024-11-17T20:26:01,618 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ca3813db0fd868a85c94bd0134efe1f/C of 6ca3813db0fd868a85c94bd0134efe1f into 0e406c2dec76403eb647711cd09f4848(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:01,618 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ca3813db0fd868a85c94bd0134efe1f: 2024-11-17T20:26:01,618 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., storeName=6ca3813db0fd868a85c94bd0134efe1f/C, priority=12, startTime=1731875160761; duration=0sec 2024-11-17T20:26:01,618 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:01,618 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ca3813db0fd868a85c94bd0134efe1f:C 2024-11-17T20:26:02,095 INFO [Thread-2054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2054. 2024-11-17T20:26:02,095 DEBUG [Thread-2054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:02,095 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:02,095 INFO [Thread-2058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2058. 2024-11-17T20:26:02,095 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:02,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:02,095 DEBUG [Thread-2058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:02,096 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:02,096 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:02,096 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:02,096 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1588 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4764 rows 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1575 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4725 rows 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1580 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4739 rows 2024-11-17T20:26:02,097 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1581 2024-11-17T20:26:02,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4743 rows 2024-11-17T20:26:02,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1578 2024-11-17T20:26:02,098 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4734 rows 2024-11-17T20:26:02,098 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:26:02,098 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:26:02,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:02,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:02,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:02,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T20:26:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T20:26:02,102 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875162102"}]},"ts":"1731875162102"} 2024-11-17T20:26:02,103 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T20:26:02,103 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T20:26:02,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:26:02,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, UNASSIGN}] 2024-11-17T20:26:02,105 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, UNASSIGN 2024-11-17T20:26:02,106 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=6ca3813db0fd868a85c94bd0134efe1f, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:02,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, UNASSIGN because future has completed 2024-11-17T20:26:02,107 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:26:02,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:26:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T20:26:02,260 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:26:02,260 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:26:02,260 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 6ca3813db0fd868a85c94bd0134efe1f, disabling compactions & flushes 2024-11-17T20:26:02,260 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:26:02,260 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:26:02,260 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. after waiting 0 ms 2024-11-17T20:26:02,261 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:26:02,261 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(2902): Flushing 6ca3813db0fd868a85c94bd0134efe1f 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-17T20:26:02,269 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/602619b9257f42aeb2656ad4c8a956e9 is 50, key is test_row_0/A:col10/1731875162091/Put/seqid=0 2024-11-17T20:26:02,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742392_1568 (size=9857) 2024-11-17T20:26:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T20:26:02,674 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/602619b9257f42aeb2656ad4c8a956e9 2024-11-17T20:26:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T20:26:02,736 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/20e1a672b01341e9a69298896ab997bf is 50, key is test_row_0/B:col10/1731875162091/Put/seqid=0 2024-11-17T20:26:02,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742393_1569 (size=9857) 2024-11-17T20:26:03,140 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/20e1a672b01341e9a69298896ab997bf 2024-11-17T20:26:03,145 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/9879f9afe77b407eb4c55a300c166f3a is 50, key is test_row_0/C:col10/1731875162091/Put/seqid=0 2024-11-17T20:26:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742394_1570 (size=9857) 2024-11-17T20:26:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T20:26:03,550 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/9879f9afe77b407eb4c55a300c166f3a 2024-11-17T20:26:03,554 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/A/602619b9257f42aeb2656ad4c8a956e9 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/602619b9257f42aeb2656ad4c8a956e9 2024-11-17T20:26:03,557 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/602619b9257f42aeb2656ad4c8a956e9, entries=100, sequenceid=502, filesize=9.6 K 2024-11-17T20:26:03,558 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/B/20e1a672b01341e9a69298896ab997bf as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/20e1a672b01341e9a69298896ab997bf 2024-11-17T20:26:03,560 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/20e1a672b01341e9a69298896ab997bf, entries=100, sequenceid=502, filesize=9.6 K 2024-11-17T20:26:03,561 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/.tmp/C/9879f9afe77b407eb4c55a300c166f3a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9879f9afe77b407eb4c55a300c166f3a 2024-11-17T20:26:03,564 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9879f9afe77b407eb4c55a300c166f3a, entries=100, sequenceid=502, filesize=9.6 K 2024-11-17T20:26:03,564 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 6ca3813db0fd868a85c94bd0134efe1f in 1303ms, sequenceid=502, compaction requested=false 2024-11-17T20:26:03,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/1c680ea0d7f24eccbd038a2f5254fd1b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/22ae040c45da41f4a8562f1026e48ddb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ffd10ced321946c2b1fca1ca3d593938, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/e14fba5ce6c64f21893f6dc19fbae65e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3d968aa9736a4030a00baa200f44542e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ed5ee474a264459b2254d0e6ee0613f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9488b1e4d94d4f2d81069857937d8bb0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3e34dcc62bc94694a61183bc48268df2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/bfd09e4783744fc2990cfcf159bef510, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/638097a3079f4074ad668153e35f2245, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c0ecbdbd7fb5439dac290642a6b20d60, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/956e2cc73d5a453eac2faaf8ac36e276, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ee41afb7d7694cae98245af0a412859b] to archive 2024-11-17T20:26:03,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:26:03,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/1c680ea0d7f24eccbd038a2f5254fd1b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/1c680ea0d7f24eccbd038a2f5254fd1b 2024-11-17T20:26:03,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/22ae040c45da41f4a8562f1026e48ddb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/22ae040c45da41f4a8562f1026e48ddb 2024-11-17T20:26:03,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ffd10ced321946c2b1fca1ca3d593938 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ffd10ced321946c2b1fca1ca3d593938 2024-11-17T20:26:03,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/e14fba5ce6c64f21893f6dc19fbae65e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/e14fba5ce6c64f21893f6dc19fbae65e 2024-11-17T20:26:03,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3d968aa9736a4030a00baa200f44542e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3d968aa9736a4030a00baa200f44542e 2024-11-17T20:26:03,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ed5ee474a264459b2254d0e6ee0613f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/7ed5ee474a264459b2254d0e6ee0613f 2024-11-17T20:26:03,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9488b1e4d94d4f2d81069857937d8bb0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/9488b1e4d94d4f2d81069857937d8bb0 2024-11-17T20:26:03,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3e34dcc62bc94694a61183bc48268df2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/3e34dcc62bc94694a61183bc48268df2 2024-11-17T20:26:03,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/bfd09e4783744fc2990cfcf159bef510 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/bfd09e4783744fc2990cfcf159bef510 2024-11-17T20:26:03,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/638097a3079f4074ad668153e35f2245 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/638097a3079f4074ad668153e35f2245 2024-11-17T20:26:03,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c0ecbdbd7fb5439dac290642a6b20d60 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/c0ecbdbd7fb5439dac290642a6b20d60 2024-11-17T20:26:03,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/956e2cc73d5a453eac2faaf8ac36e276 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/956e2cc73d5a453eac2faaf8ac36e276 2024-11-17T20:26:03,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ee41afb7d7694cae98245af0a412859b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/ee41afb7d7694cae98245af0a412859b 2024-11-17T20:26:03,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/717f1b8df19648f0a0706fba45318314, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/14e0149f912f480086d62784ceb2259a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/797ac92551ca4eec942ace4fc66a06f1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/08dcb97137a44d4daa93a6b98d800e32, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/170a58747d10496cb4c9f8dda3c6440e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/35631cf2b8674ebba235e81a2f5ca7bf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/5dadd261151e435e8592d6956a1925ca, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/03dcdc7c93de4f88bac04a67ee004926, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/41fc269460ec4ba796a98113c9d81b1e, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1aea8e5932e9476183442bf4de5d5e14, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/0b5a980811cc4270a3597e50e8b32802, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/91509d414a8e4f0bb85870934f05a673, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ca2829c12d6f4a1a944f193031ec87f7] to archive 2024-11-17T20:26:03,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:26:03,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/717f1b8df19648f0a0706fba45318314 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/717f1b8df19648f0a0706fba45318314 2024-11-17T20:26:03,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/14e0149f912f480086d62784ceb2259a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/14e0149f912f480086d62784ceb2259a 2024-11-17T20:26:03,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/797ac92551ca4eec942ace4fc66a06f1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/797ac92551ca4eec942ace4fc66a06f1 2024-11-17T20:26:03,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/08dcb97137a44d4daa93a6b98d800e32 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/08dcb97137a44d4daa93a6b98d800e32 2024-11-17T20:26:03,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/170a58747d10496cb4c9f8dda3c6440e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/170a58747d10496cb4c9f8dda3c6440e 2024-11-17T20:26:03,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/35631cf2b8674ebba235e81a2f5ca7bf to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/35631cf2b8674ebba235e81a2f5ca7bf 2024-11-17T20:26:03,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/5dadd261151e435e8592d6956a1925ca to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/5dadd261151e435e8592d6956a1925ca 2024-11-17T20:26:03,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/03dcdc7c93de4f88bac04a67ee004926 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/03dcdc7c93de4f88bac04a67ee004926 2024-11-17T20:26:03,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/41fc269460ec4ba796a98113c9d81b1e to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/41fc269460ec4ba796a98113c9d81b1e 2024-11-17T20:26:03,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1aea8e5932e9476183442bf4de5d5e14 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/1aea8e5932e9476183442bf4de5d5e14 2024-11-17T20:26:03,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/0b5a980811cc4270a3597e50e8b32802 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/0b5a980811cc4270a3597e50e8b32802 2024-11-17T20:26:03,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/91509d414a8e4f0bb85870934f05a673 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/91509d414a8e4f0bb85870934f05a673 2024-11-17T20:26:03,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ca2829c12d6f4a1a944f193031ec87f7 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ca2829c12d6f4a1a944f193031ec87f7 2024-11-17T20:26:03,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e6c457e5420e4fd190259f7f6b433c24, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/16fbc3536fc94494a2d0cfc02b702cc0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/068ec0deca8048fa960b189ea82e6cde, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d9bb8ab210d94f7381b144031b0701b3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e44e02c14c5240bca610804c00043c6f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7fd31e8c137a49f8a05ba1b838098d51, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/757bb5527fdf468ba8288d0de712724f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5886a7afdf1f45329a1044a80becaf5b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b16b8ac51d84feb986343f1f6b2d127, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b4319f80e2d34e4cb45e6d609db75026, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/91a0f2845c314c8c9fb11efe93a689bc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/75f1c4d9d43b42c984321952ed77aa22, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/62955b61babd48a0bfb85c7fad3b43c9] to archive 2024-11-17T20:26:03,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:26:03,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e6c457e5420e4fd190259f7f6b433c24 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e6c457e5420e4fd190259f7f6b433c24 2024-11-17T20:26:03,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/16fbc3536fc94494a2d0cfc02b702cc0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/16fbc3536fc94494a2d0cfc02b702cc0 2024-11-17T20:26:03,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/068ec0deca8048fa960b189ea82e6cde to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/068ec0deca8048fa960b189ea82e6cde 2024-11-17T20:26:03,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d9bb8ab210d94f7381b144031b0701b3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/d9bb8ab210d94f7381b144031b0701b3 2024-11-17T20:26:03,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e44e02c14c5240bca610804c00043c6f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/e44e02c14c5240bca610804c00043c6f 2024-11-17T20:26:03,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7fd31e8c137a49f8a05ba1b838098d51 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/7fd31e8c137a49f8a05ba1b838098d51 2024-11-17T20:26:03,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/757bb5527fdf468ba8288d0de712724f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/757bb5527fdf468ba8288d0de712724f 2024-11-17T20:26:03,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5886a7afdf1f45329a1044a80becaf5b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/5886a7afdf1f45329a1044a80becaf5b 2024-11-17T20:26:03,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b16b8ac51d84feb986343f1f6b2d127 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0b16b8ac51d84feb986343f1f6b2d127 2024-11-17T20:26:03,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b4319f80e2d34e4cb45e6d609db75026 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/b4319f80e2d34e4cb45e6d609db75026 2024-11-17T20:26:03,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/91a0f2845c314c8c9fb11efe93a689bc to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/91a0f2845c314c8c9fb11efe93a689bc 2024-11-17T20:26:03,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/75f1c4d9d43b42c984321952ed77aa22 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/75f1c4d9d43b42c984321952ed77aa22 2024-11-17T20:26:03,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/62955b61babd48a0bfb85c7fad3b43c9 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/62955b61babd48a0bfb85c7fad3b43c9 2024-11-17T20:26:03,603 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/recovered.edits/505.seqid, newMaxSeqId=505, maxSeqId=1 2024-11-17T20:26:03,603 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. 2024-11-17T20:26:03,603 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 6ca3813db0fd868a85c94bd0134efe1f: Waiting for close lock at 1731875162260Running coprocessor pre-close hooks at 1731875162260Disabling compacts and flushes for region at 1731875162260Disabling writes for close at 1731875162261 (+1 ms)Obtaining lock to block concurrent updates at 1731875162261Preparing flush snapshotting stores in 6ca3813db0fd868a85c94bd0134efe1f at 1731875162261Finished memstore snapshotting TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731875162261Flushing stores of TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f. at 1731875162263 (+2 ms)Flushing 6ca3813db0fd868a85c94bd0134efe1f/A: creating writer at 1731875162263Flushing 6ca3813db0fd868a85c94bd0134efe1f/A: appending metadata at 1731875162268 (+5 ms)Flushing 6ca3813db0fd868a85c94bd0134efe1f/A: closing flushed file at 1731875162268Flushing 6ca3813db0fd868a85c94bd0134efe1f/B: creating writer at 1731875162729 (+461 ms)Flushing 6ca3813db0fd868a85c94bd0134efe1f/B: appending metadata at 1731875162735 (+6 ms)Flushing 6ca3813db0fd868a85c94bd0134efe1f/B: closing flushed file at 1731875162735Flushing 6ca3813db0fd868a85c94bd0134efe1f/C: creating writer at 1731875163143 (+408 ms)Flushing 6ca3813db0fd868a85c94bd0134efe1f/C: appending metadata at 1731875163145 (+2 ms)Flushing 6ca3813db0fd868a85c94bd0134efe1f/C: closing flushed file at 1731875163145Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79bca2d0: reopening flushed file at 1731875163554 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f7af29a: reopening flushed file at 1731875163557 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78b645af: reopening flushed file at 1731875163560 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 6ca3813db0fd868a85c94bd0134efe1f in 1303ms, sequenceid=502, compaction requested=false at 1731875163564 (+4 ms)Writing region close event to WAL at 1731875163600 (+36 ms)Running coprocessor post-close hooks at 1731875163603 (+3 ms)Closed at 1731875163603 2024-11-17T20:26:03,605 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:26:03,605 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=6ca3813db0fd868a85c94bd0134efe1f, regionState=CLOSED 2024-11-17T20:26:03,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:26:03,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-17T20:26:03,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ca3813db0fd868a85c94bd0134efe1f, server=0b0b0ae8de03,46555,1731875029029 in 1.5000 sec 2024-11-17T20:26:03,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T20:26:03,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ca3813db0fd868a85c94bd0134efe1f, UNASSIGN in 1.5040 sec 2024-11-17T20:26:03,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-17T20:26:03,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5070 sec 2024-11-17T20:26:03,612 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875163612"}]},"ts":"1731875163612"} 2024-11-17T20:26:03,613 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T20:26:03,613 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T20:26:03,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5140 sec 2024-11-17T20:26:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T20:26:04,250 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:04,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T20:26:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:04,252 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=153, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-17T20:26:04,253 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=153, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:04,255 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:26:04,257 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/recovered.edits] 2024-11-17T20:26:04,258 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0a7f1216a01f40f98770d08713dba064 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/0a7f1216a01f40f98770d08713dba064 2024-11-17T20:26:04,259 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/602619b9257f42aeb2656ad4c8a956e9 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/A/602619b9257f42aeb2656ad4c8a956e9 2024-11-17T20:26:04,260 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/20e1a672b01341e9a69298896ab997bf to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/20e1a672b01341e9a69298896ab997bf 2024-11-17T20:26:04,261 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab9bc70c53634710afe67a363ae32de2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/B/ab9bc70c53634710afe67a363ae32de2 2024-11-17T20:26:04,263 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0e406c2dec76403eb647711cd09f4848 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/0e406c2dec76403eb647711cd09f4848 2024-11-17T20:26:04,264 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9879f9afe77b407eb4c55a300c166f3a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/C/9879f9afe77b407eb4c55a300c166f3a 2024-11-17T20:26:04,267 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/recovered.edits/505.seqid to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f/recovered.edits/505.seqid 2024-11-17T20:26:04,268 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/6ca3813db0fd868a85c94bd0134efe1f 2024-11-17T20:26:04,268 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T20:26:04,270 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=153, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:04,272 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T20:26:04,273 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T20:26:04,274 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=153, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:04,274 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T20:26:04,274 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731875164274"}]},"ts":"9223372036854775807"} 2024-11-17T20:26:04,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T20:26:04,276 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6ca3813db0fd868a85c94bd0134efe1f, NAME => 'TestAcidGuarantees,,1731875136840.6ca3813db0fd868a85c94bd0134efe1f.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T20:26:04,276 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T20:26:04,276 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731875164276"}]},"ts":"9223372036854775807"} 2024-11-17T20:26:04,278 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T20:26:04,278 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=153, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:04,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-17T20:26:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-17T20:26:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T20:26:04,363 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:04,369 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=243 (was 242) - Thread LEAK? -, OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=520 (was 552), ProcessCount=11 (was 11), AvailableMemoryMB=8097 (was 8425) 2024-11-17T20:26:04,379 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=243, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=520, ProcessCount=11, AvailableMemoryMB=8096 2024-11-17T20:26:04,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:26:04,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:26:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:04,383 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T20:26:04,383 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:04,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 154 2024-11-17T20:26:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T20:26:04,384 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T20:26:04,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742395_1571 (size=959) 2024-11-17T20:26:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T20:26:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T20:26:04,796 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828 2024-11-17T20:26:04,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742396_1572 (size=53) 2024-11-17T20:26:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T20:26:05,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:26:05,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee6d6c86df174d97a57ee15dc309739b, disabling compactions & flushes 2024-11-17T20:26:05,201 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:05,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:05,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. after waiting 0 ms 2024-11-17T20:26:05,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:05,201 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:05,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee6d6c86df174d97a57ee15dc309739b: Waiting for close lock at 1731875165201Disabling compacts and flushes for region at 1731875165201Disabling writes for close at 1731875165201Writing region close event to WAL at 1731875165201Closed at 1731875165201 2024-11-17T20:26:05,202 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T20:26:05,203 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731875165202"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731875165202"}]},"ts":"1731875165202"} 2024-11-17T20:26:05,204 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T20:26:05,204 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T20:26:05,205 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875165204"}]},"ts":"1731875165204"} 2024-11-17T20:26:05,206 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T20:26:05,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, ASSIGN}] 2024-11-17T20:26:05,207 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, ASSIGN 2024-11-17T20:26:05,208 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, ASSIGN; state=OFFLINE, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=false 2024-11-17T20:26:05,358 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:05,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, ASSIGN because future has completed 2024-11-17T20:26:05,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:26:05,515 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:05,515 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7752): Opening region: {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:26:05,515 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,515 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:26:05,516 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7794): checking encryption for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,516 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7797): checking classloading for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,517 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,518 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6d6c86df174d97a57ee15dc309739b columnFamilyName A 2024-11-17T20:26:05,518 DEBUG [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:05,518 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(327): Store=ee6d6c86df174d97a57ee15dc309739b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:26:05,518 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,519 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6d6c86df174d97a57ee15dc309739b columnFamilyName B 2024-11-17T20:26:05,519 DEBUG [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T20:26:05,519 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(327): Store=ee6d6c86df174d97a57ee15dc309739b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:26:05,520 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,520 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6d6c86df174d97a57ee15dc309739b columnFamilyName C 2024-11-17T20:26:05,520 DEBUG [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:05,521 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(327): Store=ee6d6c86df174d97a57ee15dc309739b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:26:05,521 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1038): replaying wal for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,522 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,522 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,523 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1048): stopping wal replay for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,523 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1060): Cleaning up temporary data for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,523 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:26:05,524 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1093): writing seq id for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,526 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T20:26:05,526 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1114): Opened ee6d6c86df174d97a57ee15dc309739b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70919988, jitterRate=0.056790173053741455}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:26:05,526 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:05,527 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1006): Region open journal for ee6d6c86df174d97a57ee15dc309739b: Running coprocessor pre-open hook at 1731875165516Writing region info on filesystem at 1731875165516Initializing all the Stores at 1731875165516Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875165516Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875165517 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875165517Cleaning up temporary data from old regions at 1731875165523 (+6 ms)Running coprocessor post-open hooks at 1731875165526 (+3 ms)Region opened successfully at 1731875165527 (+1 ms) 2024-11-17T20:26:05,528 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., pid=156, masterSystemTime=1731875165512 2024-11-17T20:26:05,530 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:05,530 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:05,530 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=OPEN, openSeqNum=2, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:05,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:26:05,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-17T20:26:05,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 in 172 msec 2024-11-17T20:26:05,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T20:26:05,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, ASSIGN in 327 msec 2024-11-17T20:26:05,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T20:26:05,536 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875165536"}]},"ts":"1731875165536"} 2024-11-17T20:26:05,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T20:26:05,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T20:26:05,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-11-17T20:26:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T20:26:06,539 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:06,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:06,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:06,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:06,541 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9c81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:06,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7167ef22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:06,544 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:06,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:06,546 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T20:26:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b0b0ae8de03,32801,1731875028288 2024-11-17T20:26:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f9778db 2024-11-17T20:26:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T20:26:06,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T20:26:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T20:26:06,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T20:26:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=157, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:06,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742397_1573 (size=995) 2024-11-17T20:26:06,969 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-17T20:26:06,969 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-17T20:26:06,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:26:06,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, REOPEN/MOVE}] 2024-11-17T20:26:06,973 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, REOPEN/MOVE 2024-11-17T20:26:06,974 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:06,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, REOPEN/MOVE because future has completed 2024-11-17T20:26:06,975 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:26:06,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:26:07,127 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(122): Close ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,127 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:26:07,127 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1722): Closing ee6d6c86df174d97a57ee15dc309739b, disabling compactions & flushes 2024-11-17T20:26:07,127 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,127 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,127 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. after waiting 0 ms 2024-11-17T20:26:07,127 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,130 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T20:26:07,130 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,130 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1676): Region close journal for ee6d6c86df174d97a57ee15dc309739b: Waiting for close lock at 1731875167127Running coprocessor pre-close hooks at 1731875167127Disabling compacts and flushes for region at 1731875167127Disabling writes for close at 1731875167127Writing region close event to WAL at 1731875167127Running coprocessor post-close hooks at 1731875167130 (+3 ms)Closed at 1731875167130 2024-11-17T20:26:07,130 WARN [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegionServer(3299): Not adding moved region record: ee6d6c86df174d97a57ee15dc309739b to self. 2024-11-17T20:26:07,131 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(157): Closed ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=CLOSED 2024-11-17T20:26:07,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:26:07,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-17T20:26:07,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; CloseRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 in 158 msec 2024-11-17T20:26:07,135 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, REOPEN/MOVE; state=CLOSED, location=0b0b0ae8de03,46555,1731875029029; forceNewPlan=false, retain=true 2024-11-17T20:26:07,285 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=OPENING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, REOPEN/MOVE because future has completed 2024-11-17T20:26:07,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:26:07,440 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,440 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7752): Opening region: {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} 2024-11-17T20:26:07,441 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,441 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T20:26:07,441 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7794): checking encryption for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,441 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7797): checking classloading for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,442 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,442 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6d6c86df174d97a57ee15dc309739b columnFamilyName A 2024-11-17T20:26:07,443 DEBUG [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:07,443 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(327): Store=ee6d6c86df174d97a57ee15dc309739b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:26:07,444 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,444 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6d6c86df174d97a57ee15dc309739b columnFamilyName B 2024-11-17T20:26:07,444 DEBUG [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:07,444 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(327): Store=ee6d6c86df174d97a57ee15dc309739b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:26:07,445 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,445 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6d6c86df174d97a57ee15dc309739b columnFamilyName C 2024-11-17T20:26:07,445 DEBUG [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:07,445 INFO [StoreOpener-ee6d6c86df174d97a57ee15dc309739b-1 {}] regionserver.HStore(327): Store=ee6d6c86df174d97a57ee15dc309739b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T20:26:07,445 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1038): replaying wal for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,446 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,446 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,447 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1048): stopping wal replay for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,447 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1060): Cleaning up temporary data for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,447 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T20:26:07,448 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1093): writing seq id for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,449 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1114): Opened ee6d6c86df174d97a57ee15dc309739b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68038844, jitterRate=0.013857781887054443}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T20:26:07,449 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,449 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1006): Region open journal for ee6d6c86df174d97a57ee15dc309739b: Running coprocessor pre-open hook at 1731875167441Writing region info on filesystem at 1731875167441Initializing all the Stores at 1731875167441Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875167441Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875167442 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731875167442Cleaning up temporary data from old regions at 1731875167447 (+5 ms)Running coprocessor post-open hooks at 1731875167449 (+2 ms)Region opened successfully at 1731875167449 2024-11-17T20:26:07,450 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., pid=161, masterSystemTime=1731875167438 2024-11-17T20:26:07,451 DEBUG [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,451 INFO [RS_OPEN_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,451 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=OPEN, openSeqNum=5, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:26:07,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=159 2024-11-17T20:26:07,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=159, state=SUCCESS, hasLock=false; OpenRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 in 166 msec 2024-11-17T20:26:07,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T20:26:07,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, REOPEN/MOVE in 481 msec 2024-11-17T20:26:07,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-17T20:26:07,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 485 msec 2024-11-17T20:26:07,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 899 msec 2024-11-17T20:26:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-11-17T20:26:07,459 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:07,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,461 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45fdc442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,462 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5737ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,463 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c03260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24332d1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,466 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ca8989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,467 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eebd542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,469 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a490df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,470 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747ef766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,471 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa2784f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,472 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1113bcb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,474 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b891dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,475 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@153f2586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,477 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad27f68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,478 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e5a360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,479 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fa157d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,481 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f70665a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,482 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227fce02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598ef163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b0b0ae8de03,32801,-1 for getting cluster id 2024-11-17T20:26:07,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T20:26:07,485 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd44a8168-0c44-49be-a02e-28b5ba4ed280' 2024-11-17T20:26:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T20:26:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d44a8168-0c44-49be-a02e-28b5ba4ed280" 2024-11-17T20:26:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33950466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b0b0ae8de03,32801,-1] 2024-11-17T20:26:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T20:26:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:07,486 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T20:26:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1282a403, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T20:26:07,487 DEBUG [Thread-2549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,487 DEBUG [Thread-2551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,487 DEBUG [Thread-2534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,487 DEBUG [Thread-2538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,487 DEBUG [Thread-2542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,487 DEBUG [Thread-2553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,488 DEBUG [Thread-2540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-17T20:26:07,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,490 DEBUG [Thread-2545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,491 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:26:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T20:26:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,491 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:26:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:26:07,492 DEBUG [Thread-2547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,499 DEBUG [Thread-2536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T20:26:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b0b0ae8de03,46555,1731875029029, seqNum=-1] 2024-11-17T20:26:07,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T20:26:07,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T20:26:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5] 2024-11-17T20:26:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:07,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-17T20:26:07,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875227523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875227524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175818b906ae2c4a7fbb6144ebafaffda2_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875167504/Put/seqid=0 2024-11-17T20:26:07,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875227524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875227526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742398_1574 (size=17034) 2024-11-17T20:26:07,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875227529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T20:26:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875227639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875227639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875227640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875227640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875227640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:07,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:07,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T20:26:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875227849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875227850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875227850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875227850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875227850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:07,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:07,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175818b906ae2c4a7fbb6144ebafaffda2_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175818b906ae2c4a7fbb6144ebafaffda2_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:07,936 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5a26913523ed49f89265919347a0ef08, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:07,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5a26913523ed49f89265919347a0ef08 is 175, key is test_row_0/A:col10/1731875167504/Put/seqid=0 2024-11-17T20:26:07,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742399_1575 (size=48139) 2024-11-17T20:26:07,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:07,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:07,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:07,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:08,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:08,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T20:26:08,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875228159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875228159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875228160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875228160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875228160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:08,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5a26913523ed49f89265919347a0ef08 2024-11-17T20:26:08,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/866af17cbb734489a216bbf21b53e530 is 50, key is test_row_0/B:col10/1731875167504/Put/seqid=0 2024-11-17T20:26:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742400_1576 (size=12001) 2024-11-17T20:26:08,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:08,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T20:26:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875228669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875228669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875228669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875228670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875228670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:08,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:08,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:08,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/866af17cbb734489a216bbf21b53e530 2024-11-17T20:26:08,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/9e579db68f5a4f68a6ff5707707df8c1 is 50, key is test_row_0/C:col10/1731875167504/Put/seqid=0 2024-11-17T20:26:08,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742401_1577 (size=12001) 2024-11-17T20:26:08,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:08,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:08,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:08,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:09,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:09,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:09,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:09,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:09,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:09,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:09,126 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T20:26:09,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:09,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:09,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:09,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:09,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:09,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/9e579db68f5a4f68a6ff5707707df8c1 2024-11-17T20:26:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5a26913523ed49f89265919347a0ef08 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5a26913523ed49f89265919347a0ef08 2024-11-17T20:26:09,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5a26913523ed49f89265919347a0ef08, entries=250, sequenceid=19, filesize=47.0 K 2024-11-17T20:26:09,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/866af17cbb734489a216bbf21b53e530 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/866af17cbb734489a216bbf21b53e530 2024-11-17T20:26:09,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/866af17cbb734489a216bbf21b53e530, entries=150, sequenceid=19, filesize=11.7 K 2024-11-17T20:26:09,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/9e579db68f5a4f68a6ff5707707df8c1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9e579db68f5a4f68a6ff5707707df8c1 2024-11-17T20:26:09,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9e579db68f5a4f68a6ff5707707df8c1, entries=150, sequenceid=19, filesize=11.7 K 2024-11-17T20:26:09,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ee6d6c86df174d97a57ee15dc309739b in 1719ms, sequenceid=19, compaction requested=false 2024-11-17T20:26:09,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:09,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T20:26:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:09,324 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-17T20:26:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117106747187c9d457b9b89cdfda36b5bd2_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875167525/Put/seqid=0 2024-11-17T20:26:09,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742402_1578 (size=12154) 2024-11-17T20:26:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T20:26:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875229693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875229694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875229694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875229696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875229697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:09,749 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117106747187c9d457b9b89cdfda36b5bd2_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117106747187c9d457b9b89cdfda36b5bd2_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:09,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/19d50c51703e4331bca9c63706542097, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:09,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/19d50c51703e4331bca9c63706542097 is 175, key is test_row_0/A:col10/1731875167525/Put/seqid=0 2024-11-17T20:26:09,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742403_1579 (size=30955) 2024-11-17T20:26:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875229809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875229809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875229809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875229810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875229810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875230021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875230021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875230021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875230021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875230021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,179 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/19d50c51703e4331bca9c63706542097 2024-11-17T20:26:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c7bd5b6cb18b4d79890a82ceac699117 is 50, key is test_row_0/B:col10/1731875167525/Put/seqid=0 2024-11-17T20:26:10,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742404_1580 (size=12001) 2024-11-17T20:26:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875230330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875230330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875230330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875230330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875230330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,602 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c7bd5b6cb18b4d79890a82ceac699117 2024-11-17T20:26:10,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/6c287dbaa145498b955ab728d2f708d4 is 50, key is test_row_0/C:col10/1731875167525/Put/seqid=0 2024-11-17T20:26:10,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742405_1581 (size=12001) 2024-11-17T20:26:10,619 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/6c287dbaa145498b955ab728d2f708d4 2024-11-17T20:26:10,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/19d50c51703e4331bca9c63706542097 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/19d50c51703e4331bca9c63706542097 2024-11-17T20:26:10,628 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/19d50c51703e4331bca9c63706542097, entries=150, sequenceid=40, filesize=30.2 K 2024-11-17T20:26:10,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c7bd5b6cb18b4d79890a82ceac699117 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c7bd5b6cb18b4d79890a82ceac699117 2024-11-17T20:26:10,633 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c7bd5b6cb18b4d79890a82ceac699117, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T20:26:10,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/6c287dbaa145498b955ab728d2f708d4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6c287dbaa145498b955ab728d2f708d4 2024-11-17T20:26:10,638 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6c287dbaa145498b955ab728d2f708d4, entries=150, sequenceid=40, filesize=11.7 K 2024-11-17T20:26:10,639 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ee6d6c86df174d97a57ee15dc309739b in 1315ms, sequenceid=40, compaction requested=false 2024-11-17T20:26:10,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:10,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:10,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-17T20:26:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-17T20:26:10,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T20:26:10,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1490 sec 2024-11-17T20:26:10,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.1550 sec 2024-11-17T20:26:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:10,842 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:10,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-17T20:26:10,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dec09002d1874e2e91a3e9ee169671a3_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875170842/Put/seqid=0 2024-11-17T20:26:10,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742406_1582 (size=12154) 2024-11-17T20:26:10,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:10,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875230864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dec09002d1874e2e91a3e9ee169671a3_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dec09002d1874e2e91a3e9ee169671a3_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:10,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/53a367aeeecf4649ac0f8394bfb5c27d, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:10,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/53a367aeeecf4649ac0f8394bfb5c27d is 175, key is test_row_0/A:col10/1731875170842/Put/seqid=0 2024-11-17T20:26:10,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875230869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875230870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875230871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875230872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742407_1583 (size=30955) 2024-11-17T20:26:10,894 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/53a367aeeecf4649ac0f8394bfb5c27d 2024-11-17T20:26:10,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/270189369dbc43cd8d8e6ebb0407ada5 is 50, key is test_row_0/B:col10/1731875170842/Put/seqid=0 2024-11-17T20:26:10,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742408_1584 (size=12001) 2024-11-17T20:26:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875230979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875230979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875230980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875230980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875230980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875231190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875231190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875231190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875231190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875231190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/270189369dbc43cd8d8e6ebb0407ada5 2024-11-17T20:26:11,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/0b912d7d59014dcf807577b69af0653f is 50, key is test_row_0/C:col10/1731875170842/Put/seqid=0 2024-11-17T20:26:11,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742409_1585 (size=12001) 2024-11-17T20:26:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875231504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875231505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875231505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875231505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875231506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T20:26:11,660 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:11,660 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:26:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-17T20:26:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T20:26:11,662 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:26:11,662 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:26:11,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:26:11,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/0b912d7d59014dcf807577b69af0653f 2024-11-17T20:26:11,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/53a367aeeecf4649ac0f8394bfb5c27d as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/53a367aeeecf4649ac0f8394bfb5c27d 2024-11-17T20:26:11,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/53a367aeeecf4649ac0f8394bfb5c27d, entries=150, sequenceid=57, filesize=30.2 K 2024-11-17T20:26:11,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/270189369dbc43cd8d8e6ebb0407ada5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/270189369dbc43cd8d8e6ebb0407ada5 2024-11-17T20:26:11,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/270189369dbc43cd8d8e6ebb0407ada5, entries=150, sequenceid=57, filesize=11.7 K 2024-11-17T20:26:11,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/0b912d7d59014dcf807577b69af0653f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/0b912d7d59014dcf807577b69af0653f 2024-11-17T20:26:11,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/0b912d7d59014dcf807577b69af0653f, entries=150, sequenceid=57, filesize=11.7 K 2024-11-17T20:26:11,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for ee6d6c86df174d97a57ee15dc309739b in 904ms, sequenceid=57, compaction requested=true 2024-11-17T20:26:11,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:11,747 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:11,747 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:11,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:11,748 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:11,748 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:11,748 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/A is initiating minor compaction (all files) 2024-11-17T20:26:11,748 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/B is initiating minor compaction (all files) 2024-11-17T20:26:11,748 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/B in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:11,748 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/A in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:11,748 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/866af17cbb734489a216bbf21b53e530, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c7bd5b6cb18b4d79890a82ceac699117, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/270189369dbc43cd8d8e6ebb0407ada5] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=35.2 K 2024-11-17T20:26:11,748 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5a26913523ed49f89265919347a0ef08, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/19d50c51703e4331bca9c63706542097, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/53a367aeeecf4649ac0f8394bfb5c27d] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=107.5 K 2024-11-17T20:26:11,748 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:11,748 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5a26913523ed49f89265919347a0ef08, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/19d50c51703e4331bca9c63706542097, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/53a367aeeecf4649ac0f8394bfb5c27d] 2024-11-17T20:26:11,748 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 866af17cbb734489a216bbf21b53e530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731875167498 2024-11-17T20:26:11,749 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c7bd5b6cb18b4d79890a82ceac699117, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731875167511 2024-11-17T20:26:11,749 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a26913523ed49f89265919347a0ef08, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731875167498 2024-11-17T20:26:11,749 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 270189369dbc43cd8d8e6ebb0407ada5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731875169691 2024-11-17T20:26:11,750 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d50c51703e4331bca9c63706542097, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731875167511 2024-11-17T20:26:11,750 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a367aeeecf4649ac0f8394bfb5c27d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731875169691 2024-11-17T20:26:11,761 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T20:26:11,789 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c994deb4e189472a981f302f1239a879_ee6d6c86df174d97a57ee15dc309739b store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:11,789 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:11,789 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c5ca151f13d342d4b2dd665980b64217 is 50, key is test_row_0/B:col10/1731875170842/Put/seqid=0 2024-11-17T20:26:11,791 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c994deb4e189472a981f302f1239a879_ee6d6c86df174d97a57ee15dc309739b, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:11,791 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c994deb4e189472a981f302f1239a879_ee6d6c86df174d97a57ee15dc309739b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:11,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742410_1586 (size=12104) 2024-11-17T20:26:11,809 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c5ca151f13d342d4b2dd665980b64217 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c5ca151f13d342d4b2dd665980b64217 2024-11-17T20:26:11,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T20:26:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:11,816 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-17T20:26:11,821 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/B of ee6d6c86df174d97a57ee15dc309739b into c5ca151f13d342d4b2dd665980b64217(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:11,821 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:11,821 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/B, priority=13, startTime=1731875171747; duration=0sec 2024-11-17T20:26:11,821 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:11,821 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:B 2024-11-17T20:26:11,821 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:11,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742411_1587 (size=4469) 2024-11-17T20:26:11,824 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:11,824 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/C is initiating minor compaction (all files) 2024-11-17T20:26:11,824 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/C in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:11,825 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9e579db68f5a4f68a6ff5707707df8c1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6c287dbaa145498b955ab728d2f708d4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/0b912d7d59014dcf807577b69af0653f] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=35.2 K 2024-11-17T20:26:11,826 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e579db68f5a4f68a6ff5707707df8c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731875167498 2024-11-17T20:26:11,826 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c287dbaa145498b955ab728d2f708d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731875167511 2024-11-17T20:26:11,826 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b912d7d59014dcf807577b69af0653f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731875169691 2024-11-17T20:26:11,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117074e0c19805345a9b7362354f25943d4_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875170860/Put/seqid=0 2024-11-17T20:26:11,839 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:11,840 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/1f3cd530ac634f49b66aeb12f32ae7f4 is 50, key is test_row_0/C:col10/1731875170842/Put/seqid=0 2024-11-17T20:26:11,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742413_1589 (size=12104) 2024-11-17T20:26:11,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742412_1588 (size=12154) 2024-11-17T20:26:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,886 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117074e0c19805345a9b7362354f25943d4_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117074e0c19805345a9b7362354f25943d4_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:11,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/d66717b6b34c4f9ab5a6b0643a13960b, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:11,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/d66717b6b34c4f9ab5a6b0643a13960b is 175, key is test_row_0/A:col10/1731875170860/Put/seqid=0 2024-11-17T20:26:11,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742414_1590 (size=30955) 2024-11-17T20:26:11,899 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/d66717b6b34c4f9ab5a6b0643a13960b 2024-11-17T20:26:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/a95e642e2cda48708e703e0e50470669 is 50, key is test_row_0/B:col10/1731875170860/Put/seqid=0 2024-11-17T20:26:11,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742415_1591 (size=12001) 2024-11-17T20:26:11,933 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/a95e642e2cda48708e703e0e50470669 2024-11-17T20:26:11,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/ff4e69c524ca486b9a0310aaaf1a3f85 is 50, key is test_row_0/C:col10/1731875170860/Put/seqid=0 2024-11-17T20:26:11,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742416_1592 (size=12001) 2024-11-17T20:26:11,958 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/ff4e69c524ca486b9a0310aaaf1a3f85 2024-11-17T20:26:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/d66717b6b34c4f9ab5a6b0643a13960b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/d66717b6b34c4f9ab5a6b0643a13960b 2024-11-17T20:26:11,968 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/d66717b6b34c4f9ab5a6b0643a13960b, entries=150, sequenceid=77, filesize=30.2 K 2024-11-17T20:26:11,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/a95e642e2cda48708e703e0e50470669 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a95e642e2cda48708e703e0e50470669 2024-11-17T20:26:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,973 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a95e642e2cda48708e703e0e50470669, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T20:26:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/ff4e69c524ca486b9a0310aaaf1a3f85 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ff4e69c524ca486b9a0310aaaf1a3f85 2024-11-17T20:26:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T20:26:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,980 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ff4e69c524ca486b9a0310aaaf1a3f85, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T20:26:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,981 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ee6d6c86df174d97a57ee15dc309739b in 166ms, sequenceid=77, compaction requested=false 2024-11-17T20:26:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:11,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-17T20:26:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-17T20:26:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T20:26:11,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 321 msec 2024-11-17T20:26:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 325 msec 2024-11-17T20:26:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:12,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:26:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117112a63072e084eaf9932f50224c70fd9_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875172043/Put/seqid=0 2024-11-17T20:26:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875232080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875232080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875232081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875232089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875232089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742417_1593 (size=21918) 2024-11-17T20:26:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875232199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875232200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875232200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875232200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875232200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,225 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#A#compaction#499 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:12,225 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/bf84827926d84dcfb6a7ac868124141f is 175, key is test_row_0/A:col10/1731875170842/Put/seqid=0 2024-11-17T20:26:12,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742418_1594 (size=31058) 2024-11-17T20:26:12,280 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/1f3cd530ac634f49b66aeb12f32ae7f4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/1f3cd530ac634f49b66aeb12f32ae7f4 2024-11-17T20:26:12,283 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/C of ee6d6c86df174d97a57ee15dc309739b into 1f3cd530ac634f49b66aeb12f32ae7f4(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:12,283 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:12,284 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/C, priority=13, startTime=1731875171747; duration=0sec 2024-11-17T20:26:12,284 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:12,284 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:C 2024-11-17T20:26:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T20:26:12,290 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:12,290 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:26:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-17T20:26:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T20:26:12,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:26:12,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:26:12,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:26:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T20:26:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875232410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875232410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875232410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875232410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875232410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,518 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:12,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117112a63072e084eaf9932f50224c70fd9_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117112a63072e084eaf9932f50224c70fd9_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:12,521 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/b321fb974c294002ba48f8cd9135caba, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:12,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/b321fb974c294002ba48f8cd9135caba is 175, key is test_row_0/A:col10/1731875172043/Put/seqid=0 2024-11-17T20:26:12,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742419_1595 (size=65323) 2024-11-17T20:26:12,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T20:26:12,632 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/bf84827926d84dcfb6a7ac868124141f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/bf84827926d84dcfb6a7ac868124141f 2024-11-17T20:26:12,635 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/A of ee6d6c86df174d97a57ee15dc309739b into bf84827926d84dcfb6a7ac868124141f(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:12,635 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:12,635 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/A, priority=13, startTime=1731875171747; duration=0sec 2024-11-17T20:26:12,635 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:12,635 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:A 2024-11-17T20:26:12,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875232719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875232720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875232720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875232720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875232720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:12,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:12,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T20:26:12,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/b321fb974c294002ba48f8cd9135caba 2024-11-17T20:26:12,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/efb5ff3d77734202b8192fad79b9df49 is 50, key is test_row_0/B:col10/1731875172043/Put/seqid=0 2024-11-17T20:26:12,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742420_1596 (size=12001) 2024-11-17T20:26:13,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:13,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:13,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:13,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:13,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875233230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:13,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875233230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:13,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875233230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875233230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875233230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:13,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/efb5ff3d77734202b8192fad79b9df49 2024-11-17T20:26:13,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/4ddd90d30e1345e79a12b85ea49f37aa is 50, key is test_row_0/C:col10/1731875172043/Put/seqid=0 2024-11-17T20:26:13,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742421_1597 (size=12001) 2024-11-17T20:26:13,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T20:26:13,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:13,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:13,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/4ddd90d30e1345e79a12b85ea49f37aa 2024-11-17T20:26:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/b321fb974c294002ba48f8cd9135caba as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b321fb974c294002ba48f8cd9135caba 2024-11-17T20:26:13,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b321fb974c294002ba48f8cd9135caba, entries=350, sequenceid=89, filesize=63.8 K 2024-11-17T20:26:13,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/efb5ff3d77734202b8192fad79b9df49 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/efb5ff3d77734202b8192fad79b9df49 2024-11-17T20:26:13,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/efb5ff3d77734202b8192fad79b9df49, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T20:26:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/4ddd90d30e1345e79a12b85ea49f37aa as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/4ddd90d30e1345e79a12b85ea49f37aa 2024-11-17T20:26:13,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/4ddd90d30e1345e79a12b85ea49f37aa, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T20:26:13,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee6d6c86df174d97a57ee15dc309739b in 1714ms, sequenceid=89, compaction requested=true 2024-11-17T20:26:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:13,759 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T20:26:13,759 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:13,760 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:13,760 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:13,760 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/B is initiating minor compaction (all files) 2024-11-17T20:26:13,760 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/A is initiating minor compaction (all files) 2024-11-17T20:26:13,760 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/A in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,760 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/B in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,760 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c5ca151f13d342d4b2dd665980b64217, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a95e642e2cda48708e703e0e50470669, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/efb5ff3d77734202b8192fad79b9df49] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=35.3 K 2024-11-17T20:26:13,760 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/bf84827926d84dcfb6a7ac868124141f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/d66717b6b34c4f9ab5a6b0643a13960b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b321fb974c294002ba48f8cd9135caba] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=124.4 K 2024-11-17T20:26:13,760 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,760 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/bf84827926d84dcfb6a7ac868124141f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/d66717b6b34c4f9ab5a6b0643a13960b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b321fb974c294002ba48f8cd9135caba] 2024-11-17T20:26:13,761 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ca151f13d342d4b2dd665980b64217, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731875169691 2024-11-17T20:26:13,761 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf84827926d84dcfb6a7ac868124141f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731875169691 2024-11-17T20:26:13,761 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a95e642e2cda48708e703e0e50470669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875170860 2024-11-17T20:26:13,762 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d66717b6b34c4f9ab5a6b0643a13960b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875170860 2024-11-17T20:26:13,762 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting efb5ff3d77734202b8192fad79b9df49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731875172039 2024-11-17T20:26:13,762 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b321fb974c294002ba48f8cd9135caba, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731875172029 2024-11-17T20:26:13,767 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:13,767 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/3fefd01cfdca4925baa25e3ded590071 is 50, key is test_row_0/B:col10/1731875172043/Put/seqid=0 2024-11-17T20:26:13,773 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:13,792 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d6f6325a75bc4bdc976859a13b42c98d_ee6d6c86df174d97a57ee15dc309739b store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:13,794 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d6f6325a75bc4bdc976859a13b42c98d_ee6d6c86df174d97a57ee15dc309739b, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:13,794 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d6f6325a75bc4bdc976859a13b42c98d_ee6d6c86df174d97a57ee15dc309739b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:13,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742422_1598 (size=12207) 2024-11-17T20:26:13,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742423_1599 (size=4469) 2024-11-17T20:26:13,810 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#A#compaction#509 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:13,811 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/3e77fa84061d44f9a7a5577611d15537 is 175, key is test_row_0/A:col10/1731875172043/Put/seqid=0 2024-11-17T20:26:13,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T20:26:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,818 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:26:13,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ea1a3d5146764d428299c0a14e9df0c9_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875172079/Put/seqid=0 2024-11-17T20:26:13,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742424_1600 (size=31161) 2024-11-17T20:26:13,841 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/3e77fa84061d44f9a7a5577611d15537 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/3e77fa84061d44f9a7a5577611d15537 2024-11-17T20:26:13,845 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/A of ee6d6c86df174d97a57ee15dc309739b into 3e77fa84061d44f9a7a5577611d15537(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:13,845 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:13,845 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/A, priority=13, startTime=1731875173759; duration=0sec 2024-11-17T20:26:13,845 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:13,845 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:A 2024-11-17T20:26:13,845 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:13,847 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:13,847 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/C is initiating minor compaction (all files) 2024-11-17T20:26:13,847 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/C in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:13,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742425_1601 (size=12154) 2024-11-17T20:26:13,847 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/1f3cd530ac634f49b66aeb12f32ae7f4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ff4e69c524ca486b9a0310aaaf1a3f85, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/4ddd90d30e1345e79a12b85ea49f37aa] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=35.3 K 2024-11-17T20:26:13,848 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f3cd530ac634f49b66aeb12f32ae7f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731875169691 2024-11-17T20:26:13,848 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff4e69c524ca486b9a0310aaaf1a3f85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731875170860 2024-11-17T20:26:13,848 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ddd90d30e1345e79a12b85ea49f37aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731875172039 2024-11-17T20:26:13,854 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#C#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:13,854 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/ddf4d6c2c8d040b091e98bb0cbc90aa6 is 50, key is test_row_0/C:col10/1731875172043/Put/seqid=0 2024-11-17T20:26:13,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742426_1602 (size=12207) 2024-11-17T20:26:14,213 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/3fefd01cfdca4925baa25e3ded590071 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/3fefd01cfdca4925baa25e3ded590071 2024-11-17T20:26:14,225 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/B of ee6d6c86df174d97a57ee15dc309739b into 3fefd01cfdca4925baa25e3ded590071(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:14,225 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:14,225 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/B, priority=13, startTime=1731875173759; duration=0sec 2024-11-17T20:26:14,225 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:14,225 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:B 2024-11-17T20:26:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:14,252 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ea1a3d5146764d428299c0a14e9df0c9_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ea1a3d5146764d428299c0a14e9df0c9_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/1e73a0f5eb3a40d78f5f56d5da7c1686, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:14,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/1e73a0f5eb3a40d78f5f56d5da7c1686 is 175, key is test_row_0/A:col10/1731875172079/Put/seqid=0 2024-11-17T20:26:14,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742427_1603 (size=30955) 2024-11-17T20:26:14,268 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/1e73a0f5eb3a40d78f5f56d5da7c1686 2024-11-17T20:26:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/1564b412ecd04aa1a0a672f1fcaa3b11 is 50, key is test_row_0/B:col10/1731875172079/Put/seqid=0 2024-11-17T20:26:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:14,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:14,297 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/ddf4d6c2c8d040b091e98bb0cbc90aa6 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ddf4d6c2c8d040b091e98bb0cbc90aa6 2024-11-17T20:26:14,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875234293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875234294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,304 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/C of ee6d6c86df174d97a57ee15dc309739b into ddf4d6c2c8d040b091e98bb0cbc90aa6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:14,304 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,305 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/C, priority=13, startTime=1731875173759; duration=0sec 2024-11-17T20:26:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,305 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:14,305 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:C 2024-11-17T20:26:14,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875234296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875234297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875234301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742428_1604 (size=12001) 2024-11-17T20:26:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875234409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875234410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875234410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875234410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875234410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T20:26:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875234621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875234621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875234621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875234621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875234621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,736 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/1564b412ecd04aa1a0a672f1fcaa3b11 2024-11-17T20:26:14,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/f8578485e64043db8d9811a8c8da0462 is 50, key is test_row_0/C:col10/1731875172079/Put/seqid=0 2024-11-17T20:26:14,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742429_1605 (size=12001) 2024-11-17T20:26:14,762 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/f8578485e64043db8d9811a8c8da0462 2024-11-17T20:26:14,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/1e73a0f5eb3a40d78f5f56d5da7c1686 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/1e73a0f5eb3a40d78f5f56d5da7c1686 2024-11-17T20:26:14,781 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/1e73a0f5eb3a40d78f5f56d5da7c1686, entries=150, sequenceid=116, filesize=30.2 K 2024-11-17T20:26:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/1564b412ecd04aa1a0a672f1fcaa3b11 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/1564b412ecd04aa1a0a672f1fcaa3b11 2024-11-17T20:26:14,788 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/1564b412ecd04aa1a0a672f1fcaa3b11, entries=150, sequenceid=116, filesize=11.7 K 2024-11-17T20:26:14,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/f8578485e64043db8d9811a8c8da0462 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f8578485e64043db8d9811a8c8da0462 2024-11-17T20:26:14,801 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f8578485e64043db8d9811a8c8da0462, entries=150, sequenceid=116, filesize=11.7 K 2024-11-17T20:26:14,814 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ee6d6c86df174d97a57ee15dc309739b in 996ms, sequenceid=116, compaction requested=false 2024-11-17T20:26:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-17T20:26:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-17T20:26:14,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-17T20:26:14,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5230 sec 2024-11-17T20:26:14,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.5350 sec 2024-11-17T20:26:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:14,933 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:14,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:26:14,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e7b412c0e344bc69b572b2369c5ef13_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875174933/Put/seqid=0 2024-11-17T20:26:14,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742430_1606 (size=12254) 2024-11-17T20:26:14,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:14,976 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e7b412c0e344bc69b572b2369c5ef13_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e7b412c0e344bc69b572b2369c5ef13_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:14,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/0de60add1159451faca3dd04beadb489, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:14,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/0de60add1159451faca3dd04beadb489 is 175, key is test_row_0/A:col10/1731875174933/Put/seqid=0 2024-11-17T20:26:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875234979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875234981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875234986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875234988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875234988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742431_1607 (size=31055) 2024-11-17T20:26:15,019 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/0de60add1159451faca3dd04beadb489 2024-11-17T20:26:15,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/f59017db090749eaaaf4b9d65474ab77 is 50, key is test_row_0/B:col10/1731875174933/Put/seqid=0 2024-11-17T20:26:15,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875235090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875235090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875235100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875235100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875235101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742432_1608 (size=12101) 2024-11-17T20:26:15,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875235300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875235300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875235309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875235309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875235309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/f59017db090749eaaaf4b9d65474ab77 2024-11-17T20:26:15,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/f653758b553849e086a87b47716a80c0 is 50, key is test_row_0/C:col10/1731875174933/Put/seqid=0 2024-11-17T20:26:15,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742433_1609 (size=12101) 2024-11-17T20:26:15,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/f653758b553849e086a87b47716a80c0 2024-11-17T20:26:15,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875235610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875235610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/0de60add1159451faca3dd04beadb489 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/0de60add1159451faca3dd04beadb489 2024-11-17T20:26:15,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/0de60add1159451faca3dd04beadb489, entries=150, sequenceid=132, filesize=30.3 K 2024-11-17T20:26:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875235622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875235622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875235622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:15,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/f59017db090749eaaaf4b9d65474ab77 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/f59017db090749eaaaf4b9d65474ab77 2024-11-17T20:26:15,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/f59017db090749eaaaf4b9d65474ab77, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T20:26:15,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/f653758b553849e086a87b47716a80c0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f653758b553849e086a87b47716a80c0 2024-11-17T20:26:15,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f653758b553849e086a87b47716a80c0, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T20:26:15,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for ee6d6c86df174d97a57ee15dc309739b in 713ms, sequenceid=132, compaction requested=true 2024-11-17T20:26:15,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:15,647 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:15,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:15,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:15,647 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:15,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:15,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:15,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:15,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:15,650 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:15,650 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/B is initiating minor compaction (all files) 2024-11-17T20:26:15,650 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/B in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:15,650 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/3fefd01cfdca4925baa25e3ded590071, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/1564b412ecd04aa1a0a672f1fcaa3b11, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/f59017db090749eaaaf4b9d65474ab77] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=35.5 K 2024-11-17T20:26:15,650 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fefd01cfdca4925baa25e3ded590071, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731875172039 2024-11-17T20:26:15,651 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 1564b412ecd04aa1a0a672f1fcaa3b11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731875172079 2024-11-17T20:26:15,651 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f59017db090749eaaaf4b9d65474ab77, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875174292 2024-11-17T20:26:15,653 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:15,653 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/A is initiating minor compaction (all files) 2024-11-17T20:26:15,653 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/A in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:15,653 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/3e77fa84061d44f9a7a5577611d15537, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/1e73a0f5eb3a40d78f5f56d5da7c1686, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/0de60add1159451faca3dd04beadb489] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=91.0 K 2024-11-17T20:26:15,653 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:15,653 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/3e77fa84061d44f9a7a5577611d15537, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/1e73a0f5eb3a40d78f5f56d5da7c1686, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/0de60add1159451faca3dd04beadb489] 2024-11-17T20:26:15,654 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e77fa84061d44f9a7a5577611d15537, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731875172039 2024-11-17T20:26:15,654 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e73a0f5eb3a40d78f5f56d5da7c1686, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731875172079 2024-11-17T20:26:15,657 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de60add1159451faca3dd04beadb489, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875174292 2024-11-17T20:26:15,685 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#B#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:15,685 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/7210beb7711c45f08ec7737dc75bcd78 is 50, key is test_row_0/B:col10/1731875174933/Put/seqid=0 2024-11-17T20:26:15,689 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:15,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742434_1610 (size=12409) 2024-11-17T20:26:15,718 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170330136bc5b64d3ea2be53b981fe526e_ee6d6c86df174d97a57ee15dc309739b store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:15,720 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170330136bc5b64d3ea2be53b981fe526e_ee6d6c86df174d97a57ee15dc309739b, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:15,720 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170330136bc5b64d3ea2be53b981fe526e_ee6d6c86df174d97a57ee15dc309739b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:15,722 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/7210beb7711c45f08ec7737dc75bcd78 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/7210beb7711c45f08ec7737dc75bcd78 2024-11-17T20:26:15,730 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/B of ee6d6c86df174d97a57ee15dc309739b into 7210beb7711c45f08ec7737dc75bcd78(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:15,730 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:15,730 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/B, priority=13, startTime=1731875175647; duration=0sec 2024-11-17T20:26:15,730 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:15,730 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:B 2024-11-17T20:26:15,730 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:15,731 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:15,731 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/C is initiating minor compaction (all files) 2024-11-17T20:26:15,731 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/C in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:15,731 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ddf4d6c2c8d040b091e98bb0cbc90aa6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f8578485e64043db8d9811a8c8da0462, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f653758b553849e086a87b47716a80c0] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=35.5 K 2024-11-17T20:26:15,733 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf4d6c2c8d040b091e98bb0cbc90aa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731875172039 2024-11-17T20:26:15,733 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f8578485e64043db8d9811a8c8da0462, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731875172079 2024-11-17T20:26:15,735 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f653758b553849e086a87b47716a80c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875174292 2024-11-17T20:26:15,765 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:15,766 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/6caa56892d504f6e8022c6ce66cb9da7 is 50, key is test_row_0/C:col10/1731875174933/Put/seqid=0 2024-11-17T20:26:15,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742435_1611 (size=4469) 2024-11-17T20:26:15,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742436_1612 (size=12409) 2024-11-17T20:26:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:16,124 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:16,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:26:16,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875236148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875236149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875236151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875236151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875236151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,169 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#A#compaction#518 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:16,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172cbaec6337c14e2e9215def8e9a82e11_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875174984/Put/seqid=0 2024-11-17T20:26:16,170 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/ca613a02d5674ceda94dbe02671c11df is 175, key is test_row_0/A:col10/1731875174933/Put/seqid=0 2024-11-17T20:26:16,189 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/6caa56892d504f6e8022c6ce66cb9da7 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6caa56892d504f6e8022c6ce66cb9da7 2024-11-17T20:26:16,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742437_1613 (size=31363) 2024-11-17T20:26:16,194 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/C of ee6d6c86df174d97a57ee15dc309739b into 6caa56892d504f6e8022c6ce66cb9da7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:16,194 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:16,194 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/C, priority=13, startTime=1731875175647; duration=0sec 2024-11-17T20:26:16,194 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:16,194 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:C 2024-11-17T20:26:16,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742438_1614 (size=12304) 2024-11-17T20:26:16,227 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:16,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172cbaec6337c14e2e9215def8e9a82e11_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172cbaec6337c14e2e9215def8e9a82e11_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:16,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5479864467144555aa89c1fe52b3ee27, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:16,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5479864467144555aa89c1fe52b3ee27 is 175, key is test_row_0/A:col10/1731875174984/Put/seqid=0 2024-11-17T20:26:16,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875236260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875236260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875236260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875236260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875236262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742439_1615 (size=31105) 2024-11-17T20:26:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T20:26:16,469 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:16,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:26:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-17T20:26:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T20:26:16,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:26:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875236470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875236470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875236470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875236470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,473 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:26:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:26:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875236476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T20:26:16,601 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/ca613a02d5674ceda94dbe02671c11df as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ca613a02d5674ceda94dbe02671c11df 2024-11-17T20:26:16,607 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/A of ee6d6c86df174d97a57ee15dc309739b into ca613a02d5674ceda94dbe02671c11df(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:16,607 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:16,607 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/A, priority=13, startTime=1731875175647; duration=0sec 2024-11-17T20:26:16,607 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:16,607 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:A 2024-11-17T20:26:16,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T20:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:16,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5479864467144555aa89c1fe52b3ee27 2024-11-17T20:26:16,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/679e38c3ef9047fca55f0a90a6af63f1 is 50, key is test_row_0/B:col10/1731875174984/Put/seqid=0 2024-11-17T20:26:16,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742440_1616 (size=12151) 2024-11-17T20:26:16,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T20:26:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:16,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875236779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875236779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875236779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875236780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T20:26:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875236795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:16,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T20:26:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:16,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:17,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/679e38c3ef9047fca55f0a90a6af63f1 2024-11-17T20:26:17,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T20:26:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:17,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T20:26:17,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/946e5ee9c4f84a608b30a7adb88d74cc is 50, key is test_row_0/C:col10/1731875174984/Put/seqid=0 2024-11-17T20:26:17,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742441_1617 (size=12151) 2024-11-17T20:26:17,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/946e5ee9c4f84a608b30a7adb88d74cc 2024-11-17T20:26:17,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/5479864467144555aa89c1fe52b3ee27 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5479864467144555aa89c1fe52b3ee27 2024-11-17T20:26:17,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5479864467144555aa89c1fe52b3ee27, entries=150, sequenceid=159, filesize=30.4 K 2024-11-17T20:26:17,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/679e38c3ef9047fca55f0a90a6af63f1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/679e38c3ef9047fca55f0a90a6af63f1 2024-11-17T20:26:17,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/679e38c3ef9047fca55f0a90a6af63f1, entries=150, sequenceid=159, filesize=11.9 K 2024-11-17T20:26:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/946e5ee9c4f84a608b30a7adb88d74cc as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/946e5ee9c4f84a608b30a7adb88d74cc 2024-11-17T20:26:17,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/946e5ee9c4f84a608b30a7adb88d74cc, entries=150, sequenceid=159, filesize=11.9 K 2024-11-17T20:26:17,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ee6d6c86df174d97a57ee15dc309739b in 1038ms, sequenceid=159, compaction requested=false 2024-11-17T20:26:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:17,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T20:26:17,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:17,253 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-17T20:26:17,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111796a12ac29229455ab6f06bad8c0e074e_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875176148/Put/seqid=0 2024-11-17T20:26:17,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742442_1618 (size=12304) 2024-11-17T20:26:17,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:17,268 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111796a12ac29229455ab6f06bad8c0e074e_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111796a12ac29229455ab6f06bad8c0e074e_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:17,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e82e93baa1da43619804aeafe22ceca5, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e82e93baa1da43619804aeafe22ceca5 is 175, key is test_row_0/A:col10/1731875176148/Put/seqid=0 2024-11-17T20:26:17,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742443_1619 (size=31105) 2024-11-17T20:26:17,283 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e82e93baa1da43619804aeafe22ceca5 2024-11-17T20:26:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/ae12f503232e4af8a59281e5acc0cee1 is 50, key is test_row_0/B:col10/1731875176148/Put/seqid=0 2024-11-17T20:26:17,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742444_1620 (size=12151) 2024-11-17T20:26:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:17,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875237357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875237363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875237367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875237367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875237368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T20:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875237480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875237480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875237480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875237480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875237480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T20:26:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875237690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875237690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875237690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875237690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875237691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:17,708 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/ae12f503232e4af8a59281e5acc0cee1 2024-11-17T20:26:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/04ac5b1638954da1afa7dcfa38565988 is 50, key is test_row_0/C:col10/1731875176148/Put/seqid=0 2024-11-17T20:26:17,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742445_1621 (size=12151) 2024-11-17T20:26:17,728 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/04ac5b1638954da1afa7dcfa38565988 2024-11-17T20:26:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e82e93baa1da43619804aeafe22ceca5 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e82e93baa1da43619804aeafe22ceca5 2024-11-17T20:26:17,737 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e82e93baa1da43619804aeafe22ceca5, entries=150, sequenceid=171, filesize=30.4 K 2024-11-17T20:26:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/ae12f503232e4af8a59281e5acc0cee1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ae12f503232e4af8a59281e5acc0cee1 2024-11-17T20:26:17,745 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ae12f503232e4af8a59281e5acc0cee1, entries=150, sequenceid=171, filesize=11.9 K 2024-11-17T20:26:17,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/04ac5b1638954da1afa7dcfa38565988 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/04ac5b1638954da1afa7dcfa38565988 2024-11-17T20:26:17,754 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/04ac5b1638954da1afa7dcfa38565988, entries=150, sequenceid=171, filesize=11.9 K 2024-11-17T20:26:17,755 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ee6d6c86df174d97a57ee15dc309739b in 503ms, sequenceid=171, compaction requested=true 2024-11-17T20:26:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:17,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-17T20:26:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-17T20:26:17,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-17T20:26:17,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2830 sec 2024-11-17T20:26:17,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.2880 sec 2024-11-17T20:26:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:18,002 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:18,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-17T20:26:18,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875238005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875238006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875238008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875238010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875238010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111706c04b096fbf4a878faf728c8e24a8fd_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875178001/Put/seqid=0 2024-11-17T20:26:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742446_1622 (size=12304) 2024-11-17T20:26:18,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875238121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875238121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875238121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875238121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875238121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875238330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875238330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875238330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875238330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875238330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,422 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:18,425 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111706c04b096fbf4a878faf728c8e24a8fd_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111706c04b096fbf4a878faf728c8e24a8fd_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:18,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/28fba851ba794151844ba2cf7bf798fb, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:18,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/28fba851ba794151844ba2cf7bf798fb is 175, key is test_row_0/A:col10/1731875178001/Put/seqid=0 2024-11-17T20:26:18,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742447_1623 (size=31105) 2024-11-17T20:26:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T20:26:18,620 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:18,620 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:26:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-17T20:26:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T20:26:18,622 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:26:18,623 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:26:18,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:26:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875238639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875238639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875238639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875238639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875238639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T20:26:18,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:18,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:18,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:18,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:18,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:18,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:18,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/28fba851ba794151844ba2cf7bf798fb 2024-11-17T20:26:18,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/15b7d8d4626240649c61669eba1a4161 is 50, key is test_row_0/B:col10/1731875178001/Put/seqid=0 2024-11-17T20:26:18,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742448_1624 (size=12151) 2024-11-17T20:26:18,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:18,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:18,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:18,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:18,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:18,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T20:26:19,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875239150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:19,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875239150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875239150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875239150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875239150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:19,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T20:26:19,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/15b7d8d4626240649c61669eba1a4161 2024-11-17T20:26:19,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/9ba1f387c79a499589e0159952ea8119 is 50, key is test_row_0/C:col10/1731875178001/Put/seqid=0 2024-11-17T20:26:19,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742449_1625 (size=12151) 2024-11-17T20:26:19,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:19,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:19,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/9ba1f387c79a499589e0159952ea8119 2024-11-17T20:26:19,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/28fba851ba794151844ba2cf7bf798fb as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/28fba851ba794151844ba2cf7bf798fb 2024-11-17T20:26:19,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/28fba851ba794151844ba2cf7bf798fb, entries=150, sequenceid=200, filesize=30.4 K 2024-11-17T20:26:19,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/15b7d8d4626240649c61669eba1a4161 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/15b7d8d4626240649c61669eba1a4161 2024-11-17T20:26:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/15b7d8d4626240649c61669eba1a4161, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T20:26:19,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/9ba1f387c79a499589e0159952ea8119 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9ba1f387c79a499589e0159952ea8119 2024-11-17T20:26:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9ba1f387c79a499589e0159952ea8119, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T20:26:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for ee6d6c86df174d97a57ee15dc309739b in 1728ms, sequenceid=200, compaction requested=true 2024-11-17T20:26:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:19,730 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:19,730 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,734 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124678 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:19,734 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/A is initiating minor compaction (all files) 2024-11-17T20:26:19,734 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/A in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,734 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ca613a02d5674ceda94dbe02671c11df, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5479864467144555aa89c1fe52b3ee27, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e82e93baa1da43619804aeafe22ceca5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/28fba851ba794151844ba2cf7bf798fb] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=121.8 K 2024-11-17T20:26:19,734 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,734 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ca613a02d5674ceda94dbe02671c11df, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5479864467144555aa89c1fe52b3ee27, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e82e93baa1da43619804aeafe22ceca5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/28fba851ba794151844ba2cf7bf798fb] 2024-11-17T20:26:19,734 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:19,734 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/B is initiating minor compaction (all files) 2024-11-17T20:26:19,734 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/B in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,734 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca613a02d5674ceda94dbe02671c11df, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875174292 2024-11-17T20:26:19,735 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/7210beb7711c45f08ec7737dc75bcd78, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/679e38c3ef9047fca55f0a90a6af63f1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ae12f503232e4af8a59281e5acc0cee1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/15b7d8d4626240649c61669eba1a4161] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=47.7 K 2024-11-17T20:26:19,735 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5479864467144555aa89c1fe52b3ee27, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875174977 2024-11-17T20:26:19,735 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 7210beb7711c45f08ec7737dc75bcd78, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875174292 2024-11-17T20:26:19,735 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82e93baa1da43619804aeafe22ceca5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731875176137 2024-11-17T20:26:19,735 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 679e38c3ef9047fca55f0a90a6af63f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875174977 2024-11-17T20:26:19,735 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28fba851ba794151844ba2cf7bf798fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875177363 2024-11-17T20:26:19,735 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ae12f503232e4af8a59281e5acc0cee1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731875176137 2024-11-17T20:26:19,736 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b7d8d4626240649c61669eba1a4161, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875177363 2024-11-17T20:26:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,749 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T20:26:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,761 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ab543296f6f942fd94c5016f14cc60e0_ee6d6c86df174d97a57ee15dc309739b store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,762 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#B#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,762 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/11e1b089ad854cda954ea2a06bf83bc1 is 50, key is test_row_0/B:col10/1731875178001/Put/seqid=0 2024-11-17T20:26:19,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ab543296f6f942fd94c5016f14cc60e0_ee6d6c86df174d97a57ee15dc309739b, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:19,763 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ab543296f6f942fd94c5016f14cc60e0_ee6d6c86df174d97a57ee15dc309739b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742451_1627 (size=12595) 2024-11-17T20:26:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742450_1626 (size=4469) 2024-11-17T20:26:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,782 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/11e1b089ad854cda954ea2a06bf83bc1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/11e1b089ad854cda954ea2a06bf83bc1 2024-11-17T20:26:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,789 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/B of ee6d6c86df174d97a57ee15dc309739b into 11e1b089ad854cda954ea2a06bf83bc1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:19,789 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:19,789 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/B, priority=12, startTime=1731875179730; duration=0sec 2024-11-17T20:26:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,789 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:19,789 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:B 2024-11-17T20:26:19,789 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,794 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:19,794 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/C is initiating minor compaction (all files) 2024-11-17T20:26:19,794 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/C in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,794 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6caa56892d504f6e8022c6ce66cb9da7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/946e5ee9c4f84a608b30a7adb88d74cc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/04ac5b1638954da1afa7dcfa38565988, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9ba1f387c79a499589e0159952ea8119] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=47.7 K 2024-11-17T20:26:19,795 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 6caa56892d504f6e8022c6ce66cb9da7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731875174292 2024-11-17T20:26:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,795 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 946e5ee9c4f84a608b30a7adb88d74cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731875174977 2024-11-17T20:26:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,795 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ac5b1638954da1afa7dcfa38565988, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731875176137 2024-11-17T20:26:19,795 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba1f387c79a499589e0159952ea8119, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875177363 2024-11-17T20:26:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,807 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#C#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,807 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/5360616cac6f455d8167d59756d6faba is 50, key is test_row_0/C:col10/1731875178001/Put/seqid=0 2024-11-17T20:26:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742452_1628 (size=12595) 2024-11-17T20:26:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T20:26:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:19,849 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-17T20:26:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,854 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/5360616cac6f455d8167d59756d6faba as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/5360616cac6f455d8167d59756d6faba 2024-11-17T20:26:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c249271e1c57449ab5a5bf11795d028d_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875178009/Put/seqid=0 2024-11-17T20:26:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,863 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/C of ee6d6c86df174d97a57ee15dc309739b into 5360616cac6f455d8167d59756d6faba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:19,863 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:19,863 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/C, priority=12, startTime=1731875179730; duration=0sec 2024-11-17T20:26:19,863 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:19,863 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:C 2024-11-17T20:26:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742453_1629 (size=9814) 2024-11-17T20:26:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,893 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c249271e1c57449ab5a5bf11795d028d_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c249271e1c57449ab5a5bf11795d028d_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:19,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/737e38e2151a4b959f3e4730f5e7ccab, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/737e38e2151a4b959f3e4730f5e7ccab is 175, key is test_row_0/A:col10/1731875178009/Put/seqid=0 2024-11-17T20:26:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742454_1630 (size=22461) 2024-11-17T20:26:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:20,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:20,179 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#A#compaction#529 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:20,179 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/b8a5a757633a4223aaf2e9213921e143 is 175, key is test_row_0/A:col10/1731875178001/Put/seqid=0 2024-11-17T20:26:20,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742455_1631 (size=31549) 2024-11-17T20:26:20,191 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/b8a5a757633a4223aaf2e9213921e143 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b8a5a757633a4223aaf2e9213921e143 2024-11-17T20:26:20,224 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/A of ee6d6c86df174d97a57ee15dc309739b into b8a5a757633a4223aaf2e9213921e143(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:20,224 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:20,224 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/A, priority=12, startTime=1731875179730; duration=0sec 2024-11-17T20:26:20,225 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:20,225 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:A 2024-11-17T20:26:20,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875240246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875240247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875240248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875240251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875240252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,311 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/737e38e2151a4b959f3e4730f5e7ccab 2024-11-17T20:26:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/4b6304525555445292f85a1b0588fbd0 is 50, key is test_row_0/B:col10/1731875178009/Put/seqid=0 2024-11-17T20:26:20,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742456_1632 (size=9757) 2024-11-17T20:26:20,338 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/4b6304525555445292f85a1b0588fbd0 2024-11-17T20:26:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/2f1f3930e7344961a25e968eb07545ea is 50, key is test_row_0/C:col10/1731875178009/Put/seqid=0 2024-11-17T20:26:20,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875240359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875240359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875240360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875240360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875240359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742457_1633 (size=9757) 2024-11-17T20:26:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875240570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875240570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875240570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875240570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875240571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T20:26:20,773 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/2f1f3930e7344961a25e968eb07545ea 2024-11-17T20:26:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/737e38e2151a4b959f3e4730f5e7ccab as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/737e38e2151a4b959f3e4730f5e7ccab 2024-11-17T20:26:20,780 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/737e38e2151a4b959f3e4730f5e7ccab, entries=100, sequenceid=208, filesize=21.9 K 2024-11-17T20:26:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/4b6304525555445292f85a1b0588fbd0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4b6304525555445292f85a1b0588fbd0 2024-11-17T20:26:20,785 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4b6304525555445292f85a1b0588fbd0, entries=100, sequenceid=208, filesize=9.5 K 2024-11-17T20:26:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/2f1f3930e7344961a25e968eb07545ea as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/2f1f3930e7344961a25e968eb07545ea 2024-11-17T20:26:20,790 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/2f1f3930e7344961a25e968eb07545ea, entries=100, sequenceid=208, filesize=9.5 K 2024-11-17T20:26:20,791 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=181.14 KB/185490 for ee6d6c86df174d97a57ee15dc309739b in 942ms, sequenceid=208, compaction requested=false 2024-11-17T20:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-17T20:26:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-17T20:26:20,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-17T20:26:20,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1710 sec 2024-11-17T20:26:20,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.1770 sec 2024-11-17T20:26:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:20,883 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:20,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-17T20:26:20,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875240881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875240885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875240883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875240885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875240886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117082292393df64486a5b642efc47e1ecc_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875180882/Put/seqid=0 2024-11-17T20:26:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:20,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:20,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742458_1634 (size=17284) 2024-11-17T20:26:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875240999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875241000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875241000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875241000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875241210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875241210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875241210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875241212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,303 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,306 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117082292393df64486a5b642efc47e1ecc_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117082292393df64486a5b642efc47e1ecc_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:21,307 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/ef2cadd0051049a78260393e1dbc7cb0, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:21,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/ef2cadd0051049a78260393e1dbc7cb0 is 175, key is test_row_0/A:col10/1731875180882/Put/seqid=0 2024-11-17T20:26:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742459_1635 (size=48389) 2024-11-17T20:26:21,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/ef2cadd0051049a78260393e1dbc7cb0 2024-11-17T20:26:21,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/a2ba7e70c18e4b5192c127d596bb5daa is 50, key is test_row_0/B:col10/1731875180882/Put/seqid=0 2024-11-17T20:26:21,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742460_1636 (size=12151) 2024-11-17T20:26:21,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/a2ba7e70c18e4b5192c127d596bb5daa 2024-11-17T20:26:21,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/c0062c2f3cf14c13b7bbd54b94675911 is 50, key is test_row_0/C:col10/1731875180882/Put/seqid=0 2024-11-17T20:26:21,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742461_1637 (size=12151) 2024-11-17T20:26:21,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/c0062c2f3cf14c13b7bbd54b94675911 2024-11-17T20:26:21,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/ef2cadd0051049a78260393e1dbc7cb0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ef2cadd0051049a78260393e1dbc7cb0 2024-11-17T20:26:21,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ef2cadd0051049a78260393e1dbc7cb0, entries=250, sequenceid=242, filesize=47.3 K 2024-11-17T20:26:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/a2ba7e70c18e4b5192c127d596bb5daa as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a2ba7e70c18e4b5192c127d596bb5daa 2024-11-17T20:26:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a2ba7e70c18e4b5192c127d596bb5daa, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T20:26:21,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/c0062c2f3cf14c13b7bbd54b94675911 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/c0062c2f3cf14c13b7bbd54b94675911 2024-11-17T20:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/c0062c2f3cf14c13b7bbd54b94675911, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T20:26:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for ee6d6c86df174d97a57ee15dc309739b in 503ms, sequenceid=242, compaction requested=true 2024-11-17T20:26:21,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:21,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:21,386 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T20:26:21,387 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,388 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:21,388 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/A is initiating minor compaction (all files) 2024-11-17T20:26:21,388 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/A in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:21,388 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b8a5a757633a4223aaf2e9213921e143, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/737e38e2151a4b959f3e4730f5e7ccab, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ef2cadd0051049a78260393e1dbc7cb0] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=100.0 K 2024-11-17T20:26:21,388 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:21,388 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b8a5a757633a4223aaf2e9213921e143, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/737e38e2151a4b959f3e4730f5e7ccab, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ef2cadd0051049a78260393e1dbc7cb0] 2024-11-17T20:26:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,388 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8a5a757633a4223aaf2e9213921e143, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875177363 2024-11-17T20:26:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,389 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:21,389 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737e38e2151a4b959f3e4730f5e7ccab, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731875178009 2024-11-17T20:26:21,389 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/B is initiating minor compaction (all files) 2024-11-17T20:26:21,390 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/B in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:21,390 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/11e1b089ad854cda954ea2a06bf83bc1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4b6304525555445292f85a1b0588fbd0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a2ba7e70c18e4b5192c127d596bb5daa] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=33.7 K 2024-11-17T20:26:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,390 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2cadd0051049a78260393e1dbc7cb0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731875180245 2024-11-17T20:26:21,390 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e1b089ad854cda954ea2a06bf83bc1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875177363 2024-11-17T20:26:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,390 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b6304525555445292f85a1b0588fbd0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731875178009 2024-11-17T20:26:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,391 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting a2ba7e70c18e4b5192c127d596bb5daa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731875180245 2024-11-17T20:26:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,409 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,410 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#B#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:21,410 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/43cc686175b24937a38fa70f4a783975 is 50, key is test_row_0/B:col10/1731875180882/Put/seqid=0 2024-11-17T20:26:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,423 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117809299a8202a495cad9764be8d763097_ee6d6c86df174d97a57ee15dc309739b store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:21,426 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117809299a8202a495cad9764be8d763097_ee6d6c86df174d97a57ee15dc309739b, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:21,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742462_1638 (size=12697) 2024-11-17T20:26:21,426 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117809299a8202a495cad9764be8d763097_ee6d6c86df174d97a57ee15dc309739b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,430 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/43cc686175b24937a38fa70f4a783975 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/43cc686175b24937a38fa70f4a783975 2024-11-17T20:26:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,435 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/B of ee6d6c86df174d97a57ee15dc309739b into 43cc686175b24937a38fa70f4a783975(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:21,435 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:21,435 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/B, priority=13, startTime=1731875181386; duration=0sec 2024-11-17T20:26:21,435 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:21,436 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:B 2024-11-17T20:26:21,436 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,436 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:21,436 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/C is initiating minor compaction (all files) 2024-11-17T20:26:21,436 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/C in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,437 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/5360616cac6f455d8167d59756d6faba, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/2f1f3930e7344961a25e968eb07545ea, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/c0062c2f3cf14c13b7bbd54b94675911] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=33.7 K 2024-11-17T20:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,437 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 5360616cac6f455d8167d59756d6faba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731875177363 2024-11-17T20:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,440 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f1f3930e7344961a25e968eb07545ea, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731875178009 2024-11-17T20:26:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,441 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c0062c2f3cf14c13b7bbd54b94675911, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731875180245 2024-11-17T20:26:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742463_1639 (size=4469) 2024-11-17T20:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,455 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:21,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-17T20:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:21,456 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#A#compaction#539 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:21,456 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/2b5763ea87c847efaed4fe7d4d4d0038 is 175, key is test_row_0/A:col10/1731875180882/Put/seqid=0 2024-11-17T20:26:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,470 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#C#compaction#540 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,470 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/19edede0fa7949e896aa13c84d538377 is 50, key is test_row_0/C:col10/1731875180882/Put/seqid=0 2024-11-17T20:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111715dfee6f5dbb48cf9a9c2a54133b1b6f_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875181453/Put/seqid=0 2024-11-17T20:26:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742464_1640 (size=31651) 2024-11-17T20:26:21,525 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/2b5763ea87c847efaed4fe7d4d4d0038 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2b5763ea87c847efaed4fe7d4d4d0038 2024-11-17T20:26:21,540 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/A of ee6d6c86df174d97a57ee15dc309739b into 2b5763ea87c847efaed4fe7d4d4d0038(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:21,540 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:21,540 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/A, priority=13, startTime=1731875181386; duration=0sec 2024-11-17T20:26:21,540 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:21,540 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:A 2024-11-17T20:26:21,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742465_1641 (size=12697) 2024-11-17T20:26:21,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742466_1642 (size=17284) 2024-11-17T20:26:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875241551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875241551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875241552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875241555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875241559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875241670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875241670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875241670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875241671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875241671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875241879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875241879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875241879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875241880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875241880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:21,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:21,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111715dfee6f5dbb48cf9a9c2a54133b1b6f_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111715dfee6f5dbb48cf9a9c2a54133b1b6f_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:21,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/f0af495c6c4b4fd0b7ed35c31490f4ee, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:21,960 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/19edede0fa7949e896aa13c84d538377 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/19edede0fa7949e896aa13c84d538377 2024-11-17T20:26:21,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/f0af495c6c4b4fd0b7ed35c31490f4ee is 175, key is test_row_0/A:col10/1731875181453/Put/seqid=0 2024-11-17T20:26:21,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742467_1643 (size=48389) 2024-11-17T20:26:21,964 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/C of ee6d6c86df174d97a57ee15dc309739b into 19edede0fa7949e896aa13c84d538377(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:21,964 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:21,965 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/C, priority=13, startTime=1731875181387; duration=0sec 2024-11-17T20:26:21,965 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:21,965 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:C 2024-11-17T20:26:21,965 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/f0af495c6c4b4fd0b7ed35c31490f4ee 2024-11-17T20:26:21,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/409e21508843494e9648f44f0f8e0eac is 50, key is test_row_0/B:col10/1731875181453/Put/seqid=0 2024-11-17T20:26:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742468_1644 (size=12151) 2024-11-17T20:26:22,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875242189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875242189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875242189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875242189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875242190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/409e21508843494e9648f44f0f8e0eac 2024-11-17T20:26:22,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/85d931b0b2bc416ab40ac130a35a6eef is 50, key is test_row_0/C:col10/1731875181453/Put/seqid=0 2024-11-17T20:26:22,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742469_1645 (size=12151) 2024-11-17T20:26:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875242699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875242699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875242700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875242700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875242700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T20:26:22,789 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:22,789 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:26:22,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/85d931b0b2bc416ab40ac130a35a6eef 2024-11-17T20:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-17T20:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T20:26:22,799 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:26:22,800 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:26:22,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:26:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/f0af495c6c4b4fd0b7ed35c31490f4ee as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/f0af495c6c4b4fd0b7ed35c31490f4ee 2024-11-17T20:26:22,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/f0af495c6c4b4fd0b7ed35c31490f4ee, entries=250, sequenceid=254, filesize=47.3 K 2024-11-17T20:26:22,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/409e21508843494e9648f44f0f8e0eac as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/409e21508843494e9648f44f0f8e0eac 2024-11-17T20:26:22,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/409e21508843494e9648f44f0f8e0eac, entries=150, sequenceid=254, filesize=11.9 K 2024-11-17T20:26:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/85d931b0b2bc416ab40ac130a35a6eef as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/85d931b0b2bc416ab40ac130a35a6eef 2024-11-17T20:26:22,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/85d931b0b2bc416ab40ac130a35a6eef, entries=150, sequenceid=254, filesize=11.9 K 2024-11-17T20:26:22,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee6d6c86df174d97a57ee15dc309739b in 1360ms, sequenceid=254, compaction requested=false 2024-11-17T20:26:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T20:26:22,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-17T20:26:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:22,953 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-17T20:26:22,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117028c199474404c1a8bd1f51b579a1983_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875181555/Put/seqid=0 2024-11-17T20:26:22,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742470_1646 (size=12454) 2024-11-17T20:26:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T20:26:23,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:23,364 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117028c199474404c1a8bd1f51b579a1983_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117028c199474404c1a8bd1f51b579a1983_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/40cc969937a84777adef3038a439027f, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/40cc969937a84777adef3038a439027f is 175, key is test_row_0/A:col10/1731875181555/Put/seqid=0 2024-11-17T20:26:23,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742471_1647 (size=31255) 2024-11-17T20:26:23,368 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/40cc969937a84777adef3038a439027f 2024-11-17T20:26:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/06268c9f55e1425ba0fcf68fb3e47ff1 is 50, key is test_row_0/B:col10/1731875181555/Put/seqid=0 2024-11-17T20:26:23,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742472_1648 (size=12301) 2024-11-17T20:26:23,386 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/06268c9f55e1425ba0fcf68fb3e47ff1 2024-11-17T20:26:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/d84512d1f5554ce2beaeb5a0a0056f74 is 50, key is test_row_0/C:col10/1731875181555/Put/seqid=0 2024-11-17T20:26:23,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742473_1649 (size=12301) 2024-11-17T20:26:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T20:26:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:23,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:23,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875243718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875243718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875243719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,723 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875243720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875243722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,798 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/d84512d1f5554ce2beaeb5a0a0056f74 2024-11-17T20:26:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/40cc969937a84777adef3038a439027f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/40cc969937a84777adef3038a439027f 2024-11-17T20:26:23,804 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/40cc969937a84777adef3038a439027f, entries=150, sequenceid=281, filesize=30.5 K 2024-11-17T20:26:23,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/06268c9f55e1425ba0fcf68fb3e47ff1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/06268c9f55e1425ba0fcf68fb3e47ff1 2024-11-17T20:26:23,808 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/06268c9f55e1425ba0fcf68fb3e47ff1, entries=150, sequenceid=281, filesize=12.0 K 2024-11-17T20:26:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/d84512d1f5554ce2beaeb5a0a0056f74 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/d84512d1f5554ce2beaeb5a0a0056f74 2024-11-17T20:26:23,812 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/d84512d1f5554ce2beaeb5a0a0056f74, entries=150, sequenceid=281, filesize=12.0 K 2024-11-17T20:26:23,813 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ee6d6c86df174d97a57ee15dc309739b in 861ms, sequenceid=281, compaction requested=true 2024-11-17T20:26:23,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:23,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:23,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-17T20:26:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-17T20:26:23,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-17T20:26:23,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-17T20:26:23,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.0270 sec 2024-11-17T20:26:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:23,833 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:23,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-17T20:26:23,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fbdd8966381a4d899b52d0138de56061_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875183720/Put/seqid=0 2024-11-17T20:26:23,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742474_1650 (size=12454) 2024-11-17T20:26:23,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875243863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875243863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875243865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875243867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T20:26:23,939 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:23,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T20:26:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-17T20:26:23,941 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T20:26:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T20:26:23,942 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T20:26:23,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T20:26:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875243969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875243969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875243980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875243980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T20:26:24,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875244179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875244179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875244190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875244190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:24,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,257 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T20:26:24,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fbdd8966381a4d899b52d0138de56061_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fbdd8966381a4d899b52d0138de56061_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:24,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/85d3fbf72658496e8d0e725f3b8916f4, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:24,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/85d3fbf72658496e8d0e725f3b8916f4 is 175, key is test_row_0/A:col10/1731875183720/Put/seqid=0 2024-11-17T20:26:24,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742475_1651 (size=31255) 2024-11-17T20:26:24,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875244489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875244489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875244499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875244499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:24,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T20:26:24,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/85d3fbf72658496e8d0e725f3b8916f4 2024-11-17T20:26:24,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a is 50, key is test_row_0/B:col10/1731875183720/Put/seqid=0 2024-11-17T20:26:24,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742476_1652 (size=12301) 2024-11-17T20:26:24,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:24,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875244999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875244999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:25,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875245009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:25,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:25,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875245009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:25,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T20:26:25,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a 2024-11-17T20:26:25,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/e6dddf89888d4a5f92bd90d214d3d597 is 50, key is test_row_0/C:col10/1731875183720/Put/seqid=0 2024-11-17T20:26:25,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742477_1653 (size=12301) 2024-11-17T20:26:25,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:25,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:25,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:25,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T20:26:25,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/e6dddf89888d4a5f92bd90d214d3d597 2024-11-17T20:26:25,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/85d3fbf72658496e8d0e725f3b8916f4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/85d3fbf72658496e8d0e725f3b8916f4 2024-11-17T20:26:25,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/85d3fbf72658496e8d0e725f3b8916f4, entries=150, sequenceid=294, filesize=30.5 K 2024-11-17T20:26:25,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a 2024-11-17T20:26:25,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a, entries=150, sequenceid=294, filesize=12.0 K 2024-11-17T20:26:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/e6dddf89888d4a5f92bd90d214d3d597 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/e6dddf89888d4a5f92bd90d214d3d597 2024-11-17T20:26:25,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/e6dddf89888d4a5f92bd90d214d3d597, entries=150, sequenceid=294, filesize=12.0 K 2024-11-17T20:26:25,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ee6d6c86df174d97a57ee15dc309739b in 1687ms, sequenceid=294, compaction requested=true 2024-11-17T20:26:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:25,520 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:25,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:25,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:25,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:25,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T20:26:25,521 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:25,522 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:25,522 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:25,522 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/B is initiating minor compaction (all files) 2024-11-17T20:26:25,522 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/A is initiating minor compaction (all files) 2024-11-17T20:26:25,522 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/A in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,522 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/B in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,522 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/43cc686175b24937a38fa70f4a783975, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/409e21508843494e9648f44f0f8e0eac, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/06268c9f55e1425ba0fcf68fb3e47ff1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=48.3 K 2024-11-17T20:26:25,522 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2b5763ea87c847efaed4fe7d4d4d0038, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/f0af495c6c4b4fd0b7ed35c31490f4ee, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/40cc969937a84777adef3038a439027f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/85d3fbf72658496e8d0e725f3b8916f4] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=139.2 K 2024-11-17T20:26:25,522 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,522 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2b5763ea87c847efaed4fe7d4d4d0038, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/f0af495c6c4b4fd0b7ed35c31490f4ee, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/40cc969937a84777adef3038a439027f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/85d3fbf72658496e8d0e725f3b8916f4] 2024-11-17T20:26:25,524 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43cc686175b24937a38fa70f4a783975, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731875180245 2024-11-17T20:26:25,524 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b5763ea87c847efaed4fe7d4d4d0038, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731875180245 2024-11-17T20:26:25,525 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 409e21508843494e9648f44f0f8e0eac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731875181434 2024-11-17T20:26:25,525 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting f0af495c6c4b4fd0b7ed35c31490f4ee, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731875181412 2024-11-17T20:26:25,525 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 40cc969937a84777adef3038a439027f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731875181527 2024-11-17T20:26:25,525 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06268c9f55e1425ba0fcf68fb3e47ff1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731875181527 2024-11-17T20:26:25,525 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 85d3fbf72658496e8d0e725f3b8916f4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731875183719 2024-11-17T20:26:25,525 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e1bdfbe2cf4a1c8e6ce01ed0c5323a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731875183719 2024-11-17T20:26:25,532 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#B#compaction#550 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:25,532 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c238588a16c645fabe820f47bc5657dc is 50, key is test_row_0/B:col10/1731875183720/Put/seqid=0 2024-11-17T20:26:25,533 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:25,536 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117e515e1344d1c4b06ab8102d34067b49f_ee6d6c86df174d97a57ee15dc309739b store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:25,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742478_1654 (size=12983) 2024-11-17T20:26:25,537 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117e515e1344d1c4b06ab8102d34067b49f_ee6d6c86df174d97a57ee15dc309739b, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:25,537 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e515e1344d1c4b06ab8102d34067b49f_ee6d6c86df174d97a57ee15dc309739b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:25,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742479_1655 (size=4469) 2024-11-17T20:26:25,542 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c238588a16c645fabe820f47bc5657dc as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c238588a16c645fabe820f47bc5657dc 2024-11-17T20:26:25,545 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/B of ee6d6c86df174d97a57ee15dc309739b into c238588a16c645fabe820f47bc5657dc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:25,545 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:25,546 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/B, priority=12, startTime=1731875185520; duration=0sec 2024-11-17T20:26:25,546 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:25,546 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:B 2024-11-17T20:26:25,546 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T20:26:25,546 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T20:26:25,546 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/C is initiating minor compaction (all files) 2024-11-17T20:26:25,546 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/C in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,546 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/19edede0fa7949e896aa13c84d538377, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/85d931b0b2bc416ab40ac130a35a6eef, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/d84512d1f5554ce2beaeb5a0a0056f74, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/e6dddf89888d4a5f92bd90d214d3d597] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=48.3 K 2024-11-17T20:26:25,547 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19edede0fa7949e896aa13c84d538377, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731875180245 2024-11-17T20:26:25,547 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d931b0b2bc416ab40ac130a35a6eef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731875181434 2024-11-17T20:26:25,548 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting d84512d1f5554ce2beaeb5a0a0056f74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731875181527 2024-11-17T20:26:25,548 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6dddf89888d4a5f92bd90d214d3d597, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731875183719 2024-11-17T20:26:25,553 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#C#compaction#552 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:25,553 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/317bc49a493441c2985c72b60686d45a is 50, key is test_row_0/C:col10/1731875183720/Put/seqid=0 2024-11-17T20:26:25,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742480_1656 (size=12983) 2024-11-17T20:26:25,562 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/317bc49a493441c2985c72b60686d45a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/317bc49a493441c2985c72b60686d45a 2024-11-17T20:26:25,566 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/C of ee6d6c86df174d97a57ee15dc309739b into 317bc49a493441c2985c72b60686d45a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:25,566 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:25,566 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/C, priority=12, startTime=1731875185521; duration=0sec 2024-11-17T20:26:25,566 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:25,566 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:C 2024-11-17T20:26:25,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46555 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-17T20:26:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:25,618 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T20:26:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f7170d4625434ca68655280334ff8ecd_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875183859/Put/seqid=0 2024-11-17T20:26:25,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742481_1657 (size=12454) 2024-11-17T20:26:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:25,637 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f7170d4625434ca68655280334ff8ecd_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f7170d4625434ca68655280334ff8ecd_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:25,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/11277e30e46040ce930ee465af1559d2, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:25,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/11277e30e46040ce930ee465af1559d2 is 175, key is test_row_0/A:col10/1731875183859/Put/seqid=0 2024-11-17T20:26:25,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742482_1658 (size=31255) 2024-11-17T20:26:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. as already flushing 2024-11-17T20:26:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:25,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875245823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:25,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875245929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:25,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:25,942 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#A#compaction#551 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:25,942 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/fba1e6c2728744e69f09154728c6f972 is 175, key is test_row_0/A:col10/1731875183720/Put/seqid=0 2024-11-17T20:26:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742483_1659 (size=31937) 2024-11-17T20:26:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875246019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875246019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875246019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:26,023 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:26,023 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:26,024 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875246029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:26,033 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:26,050 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/11277e30e46040ce930ee465af1559d2 2024-11-17T20:26:26,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/ba269d42251d446e8eddc8a6bd501edf is 50, key is test_row_0/B:col10/1731875183859/Put/seqid=0 2024-11-17T20:26:26,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742484_1660 (size=12301) 2024-11-17T20:26:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T20:26:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875246139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:26,352 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/fba1e6c2728744e69f09154728c6f972 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/fba1e6c2728744e69f09154728c6f972 2024-11-17T20:26:26,356 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/A of ee6d6c86df174d97a57ee15dc309739b into fba1e6c2728744e69f09154728c6f972(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:26,357 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:26,357 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/A, priority=12, startTime=1731875185520; duration=0sec 2024-11-17T20:26:26,357 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:26,357 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:A 2024-11-17T20:26:26,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875246449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:26,459 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/ba269d42251d446e8eddc8a6bd501edf 2024-11-17T20:26:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/26800c7660ca42cea45451db91791db3 is 50, key is test_row_0/C:col10/1731875183859/Put/seqid=0 2024-11-17T20:26:26,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742485_1661 (size=12301) 2024-11-17T20:26:26,467 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/26800c7660ca42cea45451db91791db3 2024-11-17T20:26:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/11277e30e46040ce930ee465af1559d2 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/11277e30e46040ce930ee465af1559d2 2024-11-17T20:26:26,482 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/11277e30e46040ce930ee465af1559d2, entries=150, sequenceid=319, filesize=30.5 K 2024-11-17T20:26:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/ba269d42251d446e8eddc8a6bd501edf as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ba269d42251d446e8eddc8a6bd501edf 2024-11-17T20:26:26,486 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ba269d42251d446e8eddc8a6bd501edf, entries=150, sequenceid=319, filesize=12.0 K 2024-11-17T20:26:26,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/26800c7660ca42cea45451db91791db3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/26800c7660ca42cea45451db91791db3 2024-11-17T20:26:26,488 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/26800c7660ca42cea45451db91791db3, entries=150, sequenceid=319, filesize=12.0 K 2024-11-17T20:26:26,497 INFO [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ee6d6c86df174d97a57ee15dc309739b in 878ms, sequenceid=319, compaction requested=false 2024-11-17T20:26:26,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:26,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:26,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b0b0ae8de03:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-17T20:26:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-17T20:26:26,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-17T20:26:26,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5560 sec 2024-11-17T20:26:26,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.5600 sec 2024-11-17T20:26:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:26,961 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:26,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-17T20:26:26,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177e26b26c02e1440db7c420f5e923436e_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875186960/Put/seqid=0 2024-11-17T20:26:26,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742486_1662 (size=14994) 2024-11-17T20:26:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875247054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875247169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:27,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:27,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177e26b26c02e1440db7c420f5e923436e_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177e26b26c02e1440db7c420f5e923436e_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:27,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e5db889c75f5425d91475dcba2f58bae, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:27,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e5db889c75f5425d91475dcba2f58bae is 175, key is test_row_0/A:col10/1731875186960/Put/seqid=0 2024-11-17T20:26:27,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742487_1663 (size=39949) 2024-11-17T20:26:27,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e5db889c75f5425d91475dcba2f58bae 2024-11-17T20:26:27,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875247379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:27,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/4ccf4c0239d846248b28a36056c7ee5f is 50, key is test_row_0/B:col10/1731875186960/Put/seqid=0 2024-11-17T20:26:27,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742488_1664 (size=12301) 2024-11-17T20:26:27,495 INFO [Thread-2553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2553. 2024-11-17T20:26:27,495 DEBUG [Thread-2553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:27,495 DEBUG [Thread-2553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,495 DEBUG [Thread-2553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:27,496 INFO [Thread-2549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2549. 2024-11-17T20:26:27,496 INFO [Thread-2551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2551. 2024-11-17T20:26:27,496 DEBUG [Thread-2549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:27,496 DEBUG [Thread-2551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:27,496 DEBUG [Thread-2549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,496 DEBUG [Thread-2551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,496 DEBUG [Thread-2549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:27,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:27,496 DEBUG [Thread-2551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,498 INFO [Thread-2545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2545. 2024-11-17T20:26:27,498 DEBUG [Thread-2545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:27,498 DEBUG [Thread-2545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,498 DEBUG [Thread-2545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:27,499 INFO [Thread-2547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2547. 2024-11-17T20:26:27,499 DEBUG [Thread-2547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:27,499 DEBUG [Thread-2547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,499 DEBUG [Thread-2547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:27,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875247690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:27,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/4ccf4c0239d846248b28a36056c7ee5f 2024-11-17T20:26:27,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/44f8c5e7175c48b79a1798372172d244 is 50, key is test_row_0/C:col10/1731875186960/Put/seqid=0 2024-11-17T20:26:27,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742489_1665 (size=12301) 2024-11-17T20:26:28,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51218 deadline: 1731875248030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:28,031 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_0' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:28,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:28,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51154 deadline: 1731875248040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51138 deadline: 1731875248040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:28,040 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_2' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:28,040 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:28,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51142 deadline: 1731875248059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:28,060 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 0b0b0ae8de03,46555,1731875029029 for 'test_row_1' in ee6d6c86df174d97a57ee15dc309739b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T20:26:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-17T20:26:28,110 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:28,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T20:26:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51160 deadline: 1731875248200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 , the old value is region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T20:26:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., hostname=0b0b0ae8de03,46555,1731875029029, seqNum=5 because the exception is null or not the one we care about 2024-11-17T20:26:28,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/44f8c5e7175c48b79a1798372172d244 2024-11-17T20:26:28,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/e5db889c75f5425d91475dcba2f58bae as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e5db889c75f5425d91475dcba2f58bae 2024-11-17T20:26:28,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e5db889c75f5425d91475dcba2f58bae, entries=200, sequenceid=334, filesize=39.0 K 2024-11-17T20:26:28,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/4ccf4c0239d846248b28a36056c7ee5f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4ccf4c0239d846248b28a36056c7ee5f 2024-11-17T20:26:28,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4ccf4c0239d846248b28a36056c7ee5f, entries=150, sequenceid=334, filesize=12.0 K 2024-11-17T20:26:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/44f8c5e7175c48b79a1798372172d244 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/44f8c5e7175c48b79a1798372172d244 2024-11-17T20:26:28,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/44f8c5e7175c48b79a1798372172d244, entries=150, sequenceid=334, filesize=12.0 K 2024-11-17T20:26:28,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ee6d6c86df174d97a57ee15dc309739b in 1270ms, sequenceid=334, compaction requested=true 2024-11-17T20:26:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:28,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T20:26:28,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:28,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T20:26:28,232 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:28,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:28,232 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:28,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee6d6c86df174d97a57ee15dc309739b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T20:26:28,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:28,232 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:28,232 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:28,232 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/A is initiating minor compaction (all files) 2024-11-17T20:26:28,232 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/B is initiating minor compaction (all files) 2024-11-17T20:26:28,233 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/A in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:28,233 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/B in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:28,233 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/fba1e6c2728744e69f09154728c6f972, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/11277e30e46040ce930ee465af1559d2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e5db889c75f5425d91475dcba2f58bae] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=100.7 K 2024-11-17T20:26:28,233 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c238588a16c645fabe820f47bc5657dc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ba269d42251d446e8eddc8a6bd501edf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4ccf4c0239d846248b28a36056c7ee5f] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=36.7 K 2024-11-17T20:26:28,233 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:28,233 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. files: [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/fba1e6c2728744e69f09154728c6f972, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/11277e30e46040ce930ee465af1559d2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e5db889c75f5425d91475dcba2f58bae] 2024-11-17T20:26:28,233 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting c238588a16c645fabe820f47bc5657dc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731875183719 2024-11-17T20:26:28,233 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba1e6c2728744e69f09154728c6f972, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731875183719 2024-11-17T20:26:28,233 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting ba269d42251d446e8eddc8a6bd501edf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731875183859 2024-11-17T20:26:28,233 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11277e30e46040ce930ee465af1559d2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731875183859 2024-11-17T20:26:28,234 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ccf4c0239d846248b28a36056c7ee5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731875185805 2024-11-17T20:26:28,234 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5db889c75f5425d91475dcba2f58bae, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731875185787 2024-11-17T20:26:28,238 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#B#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:28,239 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c230c567daf34cb990fa2d6a42dd137a is 50, key is test_row_0/B:col10/1731875186960/Put/seqid=0 2024-11-17T20:26:28,240 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:28,242 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411177d0a6b2bf7ff4dd8b3a346e92e6f4c5c_ee6d6c86df174d97a57ee15dc309739b store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:28,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742490_1666 (size=13085) 2024-11-17T20:26:28,255 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411177d0a6b2bf7ff4dd8b3a346e92e6f4c5c_ee6d6c86df174d97a57ee15dc309739b, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:28,255 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177d0a6b2bf7ff4dd8b3a346e92e6f4c5c_ee6d6c86df174d97a57ee15dc309739b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:28,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742491_1667 (size=4469) 2024-11-17T20:26:28,260 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#A#compaction#560 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:28,260 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/a609e257291c47908ff510276e2fe47a is 175, key is test_row_0/A:col10/1731875186960/Put/seqid=0 2024-11-17T20:26:28,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742492_1668 (size=32039) 2024-11-17T20:26:28,646 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/c230c567daf34cb990fa2d6a42dd137a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c230c567daf34cb990fa2d6a42dd137a 2024-11-17T20:26:28,649 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/B of ee6d6c86df174d97a57ee15dc309739b into c230c567daf34cb990fa2d6a42dd137a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:28,649 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:28,649 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/B, priority=13, startTime=1731875188232; duration=0sec 2024-11-17T20:26:28,649 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T20:26:28,649 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:B 2024-11-17T20:26:28,649 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T20:26:28,650 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T20:26:28,650 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1541): ee6d6c86df174d97a57ee15dc309739b/C is initiating minor compaction (all files) 2024-11-17T20:26:28,650 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee6d6c86df174d97a57ee15dc309739b/C in TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:28,650 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/317bc49a493441c2985c72b60686d45a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/26800c7660ca42cea45451db91791db3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/44f8c5e7175c48b79a1798372172d244] into tmpdir=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp, totalSize=36.7 K 2024-11-17T20:26:28,650 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 317bc49a493441c2985c72b60686d45a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731875183719 2024-11-17T20:26:28,651 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 26800c7660ca42cea45451db91791db3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731875183859 2024-11-17T20:26:28,651 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f8c5e7175c48b79a1798372172d244, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731875185805 2024-11-17T20:26:28,656 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee6d6c86df174d97a57ee15dc309739b#C#compaction#561 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T20:26:28,656 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/27b51ac5e1734f089f43526e244acf82 is 50, key is test_row_0/C:col10/1731875186960/Put/seqid=0 2024-11-17T20:26:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742493_1669 (size=13085) 2024-11-17T20:26:28,673 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/a609e257291c47908ff510276e2fe47a as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/a609e257291c47908ff510276e2fe47a 2024-11-17T20:26:28,676 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/A of ee6d6c86df174d97a57ee15dc309739b into a609e257291c47908ff510276e2fe47a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:28,676 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:28,676 INFO [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/A, priority=13, startTime=1731875188231; duration=0sec 2024-11-17T20:26:28,676 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:28,676 DEBUG [RS:0;0b0b0ae8de03:46555-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:A 2024-11-17T20:26:29,062 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/27b51ac5e1734f089f43526e244acf82 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/27b51ac5e1734f089f43526e244acf82 2024-11-17T20:26:29,065 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee6d6c86df174d97a57ee15dc309739b/C of ee6d6c86df174d97a57ee15dc309739b into 27b51ac5e1734f089f43526e244acf82(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T20:26:29,065 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:29,065 INFO [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., storeName=ee6d6c86df174d97a57ee15dc309739b/C, priority=13, startTime=1731875188232; duration=0sec 2024-11-17T20:26:29,065 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T20:26:29,065 DEBUG [RS:0;0b0b0ae8de03:46555-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee6d6c86df174d97a57ee15dc309739b:C 2024-11-17T20:26:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46555 {}] regionserver.HRegion(8855): Flush requested on ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:29,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-17T20:26:29,221 INFO [Thread-2540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2540. 2024-11-17T20:26:29,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-17T20:26:29,221 DEBUG [Thread-2540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:29,221 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:29,221 DEBUG [Thread-2540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:29,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:29,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a89c3b25258b4fcfb3cd64ff7df516a4_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875187053/Put/seqid=0 2024-11-17T20:26:29,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742494_1670 (size=12454) 2024-11-17T20:26:29,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:29,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a89c3b25258b4fcfb3cd64ff7df516a4_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a89c3b25258b4fcfb3cd64ff7df516a4_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:29,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/960e6827d7fe4875a612dddfbe4264d9, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:29,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/960e6827d7fe4875a612dddfbe4264d9 is 175, key is test_row_0/A:col10/1731875187053/Put/seqid=0 2024-11-17T20:26:29,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742495_1671 (size=31255) 2024-11-17T20:26:29,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/960e6827d7fe4875a612dddfbe4264d9 2024-11-17T20:26:29,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/269a8729823a4b53a07decc67c7b19d1 is 50, key is test_row_0/B:col10/1731875187053/Put/seqid=0 2024-11-17T20:26:29,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742496_1672 (size=12301) 2024-11-17T20:26:30,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/269a8729823a4b53a07decc67c7b19d1 2024-11-17T20:26:30,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/3ac2f125679d4638afab6e513e6ccef0 is 50, key is test_row_0/C:col10/1731875187053/Put/seqid=0 2024-11-17T20:26:30,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742497_1673 (size=12301) 2024-11-17T20:26:30,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/3ac2f125679d4638afab6e513e6ccef0 2024-11-17T20:26:30,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/960e6827d7fe4875a612dddfbe4264d9 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/960e6827d7fe4875a612dddfbe4264d9 2024-11-17T20:26:30,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/960e6827d7fe4875a612dddfbe4264d9, entries=150, sequenceid=360, filesize=30.5 K 2024-11-17T20:26:30,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/269a8729823a4b53a07decc67c7b19d1 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/269a8729823a4b53a07decc67c7b19d1 2024-11-17T20:26:30,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/269a8729823a4b53a07decc67c7b19d1, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T20:26:30,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/3ac2f125679d4638afab6e513e6ccef0 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/3ac2f125679d4638afab6e513e6ccef0 2024-11-17T20:26:30,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/3ac2f125679d4638afab6e513e6ccef0, entries=150, sequenceid=360, filesize=12.0 K 2024-11-17T20:26:30,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for ee6d6c86df174d97a57ee15dc309739b in 1248ms, sequenceid=360, compaction requested=false 2024-11-17T20:26:30,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee6d6c86df174d97a57ee15dc309739b: 2024-11-17T20:26:32,061 INFO [Thread-2534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2534. 2024-11-17T20:26:32,061 DEBUG [Thread-2534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:32,061 DEBUG [Thread-2534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,061 DEBUG [Thread-2534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:32,070 INFO [Thread-2538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2538. 2024-11-17T20:26:32,070 INFO [Thread-2536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2536. 2024-11-17T20:26:32,070 DEBUG [Thread-2536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:32,070 DEBUG [Thread-2538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:32,070 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,070 DEBUG [Thread-2538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,071 DEBUG [Thread-2536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,071 DEBUG [Thread-2538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:32,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:32,101 INFO [Thread-2542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2542. 2024-11-17T20:26:32,101 DEBUG [Thread-2542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T20:26:32,101 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,101 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3305 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3348 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3294 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3338 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3329 2024-11-17T20:26:32,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T20:26:32,102 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:26:32,102 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:26:32,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:32,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:32,102 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T20:26:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T20:26:32,105 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875192104"}]},"ts":"1731875192104"} 2024-11-17T20:26:32,106 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T20:26:32,106 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T20:26:32,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T20:26:32,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, UNASSIGN}] 2024-11-17T20:26:32,108 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, UNASSIGN 2024-11-17T20:26:32,109 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=CLOSING, regionLocation=0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:32,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, UNASSIGN because future has completed 2024-11-17T20:26:32,110 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T20:26:32,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029}] 2024-11-17T20:26:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T20:26:32,262 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(122): Close ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:32,262 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T20:26:32,262 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1722): Closing ee6d6c86df174d97a57ee15dc309739b, disabling compactions & flushes 2024-11-17T20:26:32,263 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:32,263 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:32,263 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. after waiting 0 ms 2024-11-17T20:26:32,263 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:32,263 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(2902): Flushing ee6d6c86df174d97a57ee15dc309739b 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-17T20:26:32,268 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730a452d65180480fa44ab8eee58e6ec2_ee6d6c86df174d97a57ee15dc309739b is 50, key is test_row_0/A:col10/1731875192069/Put/seqid=0 2024-11-17T20:26:32,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742498_1674 (size=12454) 2024-11-17T20:26:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T20:26:32,672 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T20:26:32,675 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730a452d65180480fa44ab8eee58e6ec2_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730a452d65180480fa44ab8eee58e6ec2_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:32,676 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/2411fdcf77f74c929ce71a6f865b2a17, store: [table=TestAcidGuarantees family=A region=ee6d6c86df174d97a57ee15dc309739b] 2024-11-17T20:26:32,676 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/2411fdcf77f74c929ce71a6f865b2a17 is 175, key is test_row_0/A:col10/1731875192069/Put/seqid=0 2024-11-17T20:26:32,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742499_1675 (size=31255) 2024-11-17T20:26:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T20:26:33,080 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/2411fdcf77f74c929ce71a6f865b2a17 2024-11-17T20:26:33,085 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/d56b17a5b53048478e7cee1fd0ebcc0f is 50, key is test_row_0/B:col10/1731875192069/Put/seqid=0 2024-11-17T20:26:33,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742500_1676 (size=12301) 2024-11-17T20:26:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T20:26:33,488 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/d56b17a5b53048478e7cee1fd0ebcc0f 2024-11-17T20:26:33,493 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/79058c39b991481e87c6c94ea0603ce8 is 50, key is test_row_0/C:col10/1731875192069/Put/seqid=0 2024-11-17T20:26:33,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742501_1677 (size=12301) 2024-11-17T20:26:33,896 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/79058c39b991481e87c6c94ea0603ce8 2024-11-17T20:26:33,899 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/A/2411fdcf77f74c929ce71a6f865b2a17 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2411fdcf77f74c929ce71a6f865b2a17 2024-11-17T20:26:33,902 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2411fdcf77f74c929ce71a6f865b2a17, entries=150, sequenceid=367, filesize=30.5 K 2024-11-17T20:26:33,902 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/B/d56b17a5b53048478e7cee1fd0ebcc0f as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/d56b17a5b53048478e7cee1fd0ebcc0f 2024-11-17T20:26:33,905 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/d56b17a5b53048478e7cee1fd0ebcc0f, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T20:26:33,906 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/.tmp/C/79058c39b991481e87c6c94ea0603ce8 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/79058c39b991481e87c6c94ea0603ce8 2024-11-17T20:26:33,908 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/79058c39b991481e87c6c94ea0603ce8, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T20:26:33,909 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for ee6d6c86df174d97a57ee15dc309739b in 1646ms, sequenceid=367, compaction requested=true 2024-11-17T20:26:33,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5a26913523ed49f89265919347a0ef08, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/19d50c51703e4331bca9c63706542097, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/bf84827926d84dcfb6a7ac868124141f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/53a367aeeecf4649ac0f8394bfb5c27d, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/d66717b6b34c4f9ab5a6b0643a13960b, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b321fb974c294002ba48f8cd9135caba, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/3e77fa84061d44f9a7a5577611d15537, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/1e73a0f5eb3a40d78f5f56d5da7c1686, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ca613a02d5674ceda94dbe02671c11df, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/0de60add1159451faca3dd04beadb489, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5479864467144555aa89c1fe52b3ee27, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e82e93baa1da43619804aeafe22ceca5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b8a5a757633a4223aaf2e9213921e143, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/28fba851ba794151844ba2cf7bf798fb, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/737e38e2151a4b959f3e4730f5e7ccab, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ef2cadd0051049a78260393e1dbc7cb0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2b5763ea87c847efaed4fe7d4d4d0038, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/f0af495c6c4b4fd0b7ed35c31490f4ee, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/40cc969937a84777adef3038a439027f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/fba1e6c2728744e69f09154728c6f972, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/85d3fbf72658496e8d0e725f3b8916f4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/11277e30e46040ce930ee465af1559d2, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e5db889c75f5425d91475dcba2f58bae] to archive 2024-11-17T20:26:33,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:26:33,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5a26913523ed49f89265919347a0ef08 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5a26913523ed49f89265919347a0ef08 2024-11-17T20:26:33,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/19d50c51703e4331bca9c63706542097 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/19d50c51703e4331bca9c63706542097 2024-11-17T20:26:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/bf84827926d84dcfb6a7ac868124141f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/bf84827926d84dcfb6a7ac868124141f 2024-11-17T20:26:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/53a367aeeecf4649ac0f8394bfb5c27d to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/53a367aeeecf4649ac0f8394bfb5c27d 2024-11-17T20:26:33,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/d66717b6b34c4f9ab5a6b0643a13960b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/d66717b6b34c4f9ab5a6b0643a13960b 2024-11-17T20:26:33,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b321fb974c294002ba48f8cd9135caba to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b321fb974c294002ba48f8cd9135caba 2024-11-17T20:26:33,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/3e77fa84061d44f9a7a5577611d15537 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/3e77fa84061d44f9a7a5577611d15537 2024-11-17T20:26:33,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/1e73a0f5eb3a40d78f5f56d5da7c1686 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/1e73a0f5eb3a40d78f5f56d5da7c1686 2024-11-17T20:26:33,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ca613a02d5674ceda94dbe02671c11df to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ca613a02d5674ceda94dbe02671c11df 2024-11-17T20:26:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/0de60add1159451faca3dd04beadb489 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/0de60add1159451faca3dd04beadb489 2024-11-17T20:26:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5479864467144555aa89c1fe52b3ee27 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/5479864467144555aa89c1fe52b3ee27 2024-11-17T20:26:33,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e82e93baa1da43619804aeafe22ceca5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e82e93baa1da43619804aeafe22ceca5 2024-11-17T20:26:33,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b8a5a757633a4223aaf2e9213921e143 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/b8a5a757633a4223aaf2e9213921e143 2024-11-17T20:26:33,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/28fba851ba794151844ba2cf7bf798fb to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/28fba851ba794151844ba2cf7bf798fb 2024-11-17T20:26:33,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/737e38e2151a4b959f3e4730f5e7ccab to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/737e38e2151a4b959f3e4730f5e7ccab 2024-11-17T20:26:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ef2cadd0051049a78260393e1dbc7cb0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/ef2cadd0051049a78260393e1dbc7cb0 2024-11-17T20:26:33,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2b5763ea87c847efaed4fe7d4d4d0038 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2b5763ea87c847efaed4fe7d4d4d0038 2024-11-17T20:26:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/f0af495c6c4b4fd0b7ed35c31490f4ee to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/f0af495c6c4b4fd0b7ed35c31490f4ee 2024-11-17T20:26:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/40cc969937a84777adef3038a439027f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/40cc969937a84777adef3038a439027f 2024-11-17T20:26:33,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/fba1e6c2728744e69f09154728c6f972 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/fba1e6c2728744e69f09154728c6f972 2024-11-17T20:26:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/85d3fbf72658496e8d0e725f3b8916f4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/85d3fbf72658496e8d0e725f3b8916f4 2024-11-17T20:26:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/11277e30e46040ce930ee465af1559d2 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/11277e30e46040ce930ee465af1559d2 2024-11-17T20:26:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e5db889c75f5425d91475dcba2f58bae to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/e5db889c75f5425d91475dcba2f58bae 2024-11-17T20:26:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/866af17cbb734489a216bbf21b53e530, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c7bd5b6cb18b4d79890a82ceac699117, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c5ca151f13d342d4b2dd665980b64217, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/270189369dbc43cd8d8e6ebb0407ada5, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a95e642e2cda48708e703e0e50470669, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/3fefd01cfdca4925baa25e3ded590071, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/efb5ff3d77734202b8192fad79b9df49, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/1564b412ecd04aa1a0a672f1fcaa3b11, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/7210beb7711c45f08ec7737dc75bcd78, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/f59017db090749eaaaf4b9d65474ab77, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/679e38c3ef9047fca55f0a90a6af63f1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ae12f503232e4af8a59281e5acc0cee1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/11e1b089ad854cda954ea2a06bf83bc1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/15b7d8d4626240649c61669eba1a4161, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4b6304525555445292f85a1b0588fbd0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/43cc686175b24937a38fa70f4a783975, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a2ba7e70c18e4b5192c127d596bb5daa, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/409e21508843494e9648f44f0f8e0eac, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/06268c9f55e1425ba0fcf68fb3e47ff1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c238588a16c645fabe820f47bc5657dc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ba269d42251d446e8eddc8a6bd501edf, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4ccf4c0239d846248b28a36056c7ee5f] to archive 2024-11-17T20:26:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:26:33,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/866af17cbb734489a216bbf21b53e530 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/866af17cbb734489a216bbf21b53e530 2024-11-17T20:26:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c7bd5b6cb18b4d79890a82ceac699117 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c7bd5b6cb18b4d79890a82ceac699117 2024-11-17T20:26:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c5ca151f13d342d4b2dd665980b64217 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c5ca151f13d342d4b2dd665980b64217 2024-11-17T20:26:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/270189369dbc43cd8d8e6ebb0407ada5 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/270189369dbc43cd8d8e6ebb0407ada5 2024-11-17T20:26:33,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a95e642e2cda48708e703e0e50470669 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a95e642e2cda48708e703e0e50470669 2024-11-17T20:26:33,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/3fefd01cfdca4925baa25e3ded590071 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/3fefd01cfdca4925baa25e3ded590071 2024-11-17T20:26:33,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/efb5ff3d77734202b8192fad79b9df49 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/efb5ff3d77734202b8192fad79b9df49 2024-11-17T20:26:33,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/1564b412ecd04aa1a0a672f1fcaa3b11 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/1564b412ecd04aa1a0a672f1fcaa3b11 2024-11-17T20:26:33,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/7210beb7711c45f08ec7737dc75bcd78 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/7210beb7711c45f08ec7737dc75bcd78 2024-11-17T20:26:33,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/f59017db090749eaaaf4b9d65474ab77 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/f59017db090749eaaaf4b9d65474ab77 2024-11-17T20:26:33,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/679e38c3ef9047fca55f0a90a6af63f1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/679e38c3ef9047fca55f0a90a6af63f1 2024-11-17T20:26:33,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ae12f503232e4af8a59281e5acc0cee1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ae12f503232e4af8a59281e5acc0cee1 2024-11-17T20:26:33,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/11e1b089ad854cda954ea2a06bf83bc1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/11e1b089ad854cda954ea2a06bf83bc1 2024-11-17T20:26:33,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/15b7d8d4626240649c61669eba1a4161 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/15b7d8d4626240649c61669eba1a4161 2024-11-17T20:26:33,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4b6304525555445292f85a1b0588fbd0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4b6304525555445292f85a1b0588fbd0 2024-11-17T20:26:33,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/43cc686175b24937a38fa70f4a783975 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/43cc686175b24937a38fa70f4a783975 2024-11-17T20:26:33,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a2ba7e70c18e4b5192c127d596bb5daa to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/a2ba7e70c18e4b5192c127d596bb5daa 2024-11-17T20:26:33,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/409e21508843494e9648f44f0f8e0eac to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/409e21508843494e9648f44f0f8e0eac 2024-11-17T20:26:33,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/06268c9f55e1425ba0fcf68fb3e47ff1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/06268c9f55e1425ba0fcf68fb3e47ff1 2024-11-17T20:26:33,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c238588a16c645fabe820f47bc5657dc to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c238588a16c645fabe820f47bc5657dc 2024-11-17T20:26:33,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/40e1bdfbe2cf4a1c8e6ce01ed0c5323a 2024-11-17T20:26:33,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ba269d42251d446e8eddc8a6bd501edf to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/ba269d42251d446e8eddc8a6bd501edf 2024-11-17T20:26:33,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4ccf4c0239d846248b28a36056c7ee5f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/4ccf4c0239d846248b28a36056c7ee5f 2024-11-17T20:26:33,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9e579db68f5a4f68a6ff5707707df8c1, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6c287dbaa145498b955ab728d2f708d4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/1f3cd530ac634f49b66aeb12f32ae7f4, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/0b912d7d59014dcf807577b69af0653f, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ff4e69c524ca486b9a0310aaaf1a3f85, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ddf4d6c2c8d040b091e98bb0cbc90aa6, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/4ddd90d30e1345e79a12b85ea49f37aa, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f8578485e64043db8d9811a8c8da0462, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6caa56892d504f6e8022c6ce66cb9da7, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f653758b553849e086a87b47716a80c0, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/946e5ee9c4f84a608b30a7adb88d74cc, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/04ac5b1638954da1afa7dcfa38565988, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/5360616cac6f455d8167d59756d6faba, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9ba1f387c79a499589e0159952ea8119, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/2f1f3930e7344961a25e968eb07545ea, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/19edede0fa7949e896aa13c84d538377, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/c0062c2f3cf14c13b7bbd54b94675911, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/85d931b0b2bc416ab40ac130a35a6eef, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/d84512d1f5554ce2beaeb5a0a0056f74, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/317bc49a493441c2985c72b60686d45a, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/e6dddf89888d4a5f92bd90d214d3d597, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/26800c7660ca42cea45451db91791db3, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/44f8c5e7175c48b79a1798372172d244] to archive 2024-11-17T20:26:33,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T20:26:33,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9e579db68f5a4f68a6ff5707707df8c1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9e579db68f5a4f68a6ff5707707df8c1 2024-11-17T20:26:33,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6c287dbaa145498b955ab728d2f708d4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6c287dbaa145498b955ab728d2f708d4 2024-11-17T20:26:33,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/1f3cd530ac634f49b66aeb12f32ae7f4 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/1f3cd530ac634f49b66aeb12f32ae7f4 2024-11-17T20:26:33,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/0b912d7d59014dcf807577b69af0653f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/0b912d7d59014dcf807577b69af0653f 2024-11-17T20:26:33,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ff4e69c524ca486b9a0310aaaf1a3f85 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ff4e69c524ca486b9a0310aaaf1a3f85 2024-11-17T20:26:33,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ddf4d6c2c8d040b091e98bb0cbc90aa6 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/ddf4d6c2c8d040b091e98bb0cbc90aa6 2024-11-17T20:26:33,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/4ddd90d30e1345e79a12b85ea49f37aa to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/4ddd90d30e1345e79a12b85ea49f37aa 2024-11-17T20:26:33,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f8578485e64043db8d9811a8c8da0462 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f8578485e64043db8d9811a8c8da0462 2024-11-17T20:26:33,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6caa56892d504f6e8022c6ce66cb9da7 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/6caa56892d504f6e8022c6ce66cb9da7 2024-11-17T20:26:33,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f653758b553849e086a87b47716a80c0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/f653758b553849e086a87b47716a80c0 2024-11-17T20:26:33,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/946e5ee9c4f84a608b30a7adb88d74cc to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/946e5ee9c4f84a608b30a7adb88d74cc 2024-11-17T20:26:33,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/04ac5b1638954da1afa7dcfa38565988 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/04ac5b1638954da1afa7dcfa38565988 2024-11-17T20:26:33,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/5360616cac6f455d8167d59756d6faba to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/5360616cac6f455d8167d59756d6faba 2024-11-17T20:26:33,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9ba1f387c79a499589e0159952ea8119 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/9ba1f387c79a499589e0159952ea8119 2024-11-17T20:26:33,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/2f1f3930e7344961a25e968eb07545ea to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/2f1f3930e7344961a25e968eb07545ea 2024-11-17T20:26:33,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/19edede0fa7949e896aa13c84d538377 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/19edede0fa7949e896aa13c84d538377 2024-11-17T20:26:33,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/c0062c2f3cf14c13b7bbd54b94675911 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/c0062c2f3cf14c13b7bbd54b94675911 2024-11-17T20:26:33,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/85d931b0b2bc416ab40ac130a35a6eef to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/85d931b0b2bc416ab40ac130a35a6eef 2024-11-17T20:26:33,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/d84512d1f5554ce2beaeb5a0a0056f74 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/d84512d1f5554ce2beaeb5a0a0056f74 2024-11-17T20:26:33,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/317bc49a493441c2985c72b60686d45a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/317bc49a493441c2985c72b60686d45a 2024-11-17T20:26:33,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/e6dddf89888d4a5f92bd90d214d3d597 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/e6dddf89888d4a5f92bd90d214d3d597 2024-11-17T20:26:33,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/26800c7660ca42cea45451db91791db3 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/26800c7660ca42cea45451db91791db3 2024-11-17T20:26:33,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/44f8c5e7175c48b79a1798372172d244 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/44f8c5e7175c48b79a1798372172d244 2024-11-17T20:26:33,975 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=4 2024-11-17T20:26:33,976 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. 2024-11-17T20:26:33,976 DEBUG [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1676): Region close journal for ee6d6c86df174d97a57ee15dc309739b: Waiting for close lock at 1731875192262Running coprocessor pre-close hooks at 1731875192262Disabling compacts and flushes for region at 1731875192262Disabling writes for close at 1731875192263 (+1 ms)Obtaining lock to block concurrent updates at 1731875192263Preparing flush snapshotting stores in ee6d6c86df174d97a57ee15dc309739b at 1731875192263Finished memstore snapshotting TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731875192263Flushing stores of TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b. at 1731875192264 (+1 ms)Flushing ee6d6c86df174d97a57ee15dc309739b/A: creating writer at 1731875192264Flushing mob file ee6d6c86df174d97a57ee15dc309739b/A: appending metadata at 1731875192268 (+4 ms)Flushing mob file ee6d6c86df174d97a57ee15dc309739b/A: closing flushed file at 1731875192268Flushing ee6d6c86df174d97a57ee15dc309739b/A: appending metadata at 1731875192676 (+408 ms)Flushing ee6d6c86df174d97a57ee15dc309739b/A: closing flushed file at 1731875192676Flushing ee6d6c86df174d97a57ee15dc309739b/B: creating writer at 1731875193082 (+406 ms)Flushing ee6d6c86df174d97a57ee15dc309739b/B: appending metadata at 1731875193084 (+2 ms)Flushing ee6d6c86df174d97a57ee15dc309739b/B: closing flushed file at 1731875193084Flushing ee6d6c86df174d97a57ee15dc309739b/C: creating writer at 1731875193491 (+407 ms)Flushing ee6d6c86df174d97a57ee15dc309739b/C: appending metadata at 1731875193493 (+2 ms)Flushing ee6d6c86df174d97a57ee15dc309739b/C: closing flushed file at 1731875193493Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ed1e2fb: reopening flushed file at 1731875193899 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c55e740: reopening flushed file at 1731875193902 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21adc7a3: reopening flushed file at 1731875193905 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for ee6d6c86df174d97a57ee15dc309739b in 1646ms, sequenceid=367, compaction requested=true at 1731875193909 (+4 ms)Writing region close event to WAL at 1731875193973 (+64 ms)Running coprocessor post-close hooks at 1731875193976 (+3 ms)Closed at 1731875193976 2024-11-17T20:26:33,977 INFO [RS_CLOSE_REGION-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(157): Closed ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:33,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=ee6d6c86df174d97a57ee15dc309739b, regionState=CLOSED 2024-11-17T20:26:33,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 because future has completed 2024-11-17T20:26:33,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-17T20:26:33,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseRegionProcedure ee6d6c86df174d97a57ee15dc309739b, server=0b0b0ae8de03,46555,1731875029029 in 1.8700 sec 2024-11-17T20:26:33,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=178, resume processing ppid=177 2024-11-17T20:26:33,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, ppid=177, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee6d6c86df174d97a57ee15dc309739b, UNASSIGN in 1.8740 sec 2024-11-17T20:26:33,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-17T20:26:33,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-17T20:26:33,985 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731875193985"}]},"ts":"1731875193985"} 2024-11-17T20:26:33,986 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T20:26:33,986 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T20:26:33,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8840 sec 2024-11-17T20:26:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T20:26:34,250 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:34,252 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T20:26:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:34,256 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-17T20:26:34,257 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=180, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:34,262 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,266 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C, FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/recovered.edits] 2024-11-17T20:26:34,270 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2411fdcf77f74c929ce71a6f865b2a17 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/2411fdcf77f74c929ce71a6f865b2a17 2024-11-17T20:26:34,272 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/960e6827d7fe4875a612dddfbe4264d9 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/960e6827d7fe4875a612dddfbe4264d9 2024-11-17T20:26:34,273 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/a609e257291c47908ff510276e2fe47a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/A/a609e257291c47908ff510276e2fe47a 2024-11-17T20:26:34,277 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/269a8729823a4b53a07decc67c7b19d1 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/269a8729823a4b53a07decc67c7b19d1 2024-11-17T20:26:34,279 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c230c567daf34cb990fa2d6a42dd137a to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/c230c567daf34cb990fa2d6a42dd137a 2024-11-17T20:26:34,280 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/d56b17a5b53048478e7cee1fd0ebcc0f to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/B/d56b17a5b53048478e7cee1fd0ebcc0f 2024-11-17T20:26:34,283 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/27b51ac5e1734f089f43526e244acf82 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/27b51ac5e1734f089f43526e244acf82 2024-11-17T20:26:34,284 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/3ac2f125679d4638afab6e513e6ccef0 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/3ac2f125679d4638afab6e513e6ccef0 2024-11-17T20:26:34,285 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/79058c39b991481e87c6c94ea0603ce8 to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/C/79058c39b991481e87c6c94ea0603ce8 2024-11-17T20:26:34,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/recovered.edits/370.seqid to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b/recovered.edits/370.seqid 2024-11-17T20:26:34,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/default/TestAcidGuarantees/ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,288 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T20:26:34,289 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T20:26:34,289 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T20:26:34,292 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117028c199474404c1a8bd1f51b579a1983_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117028c199474404c1a8bd1f51b579a1983_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,294 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111706c04b096fbf4a878faf728c8e24a8fd_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111706c04b096fbf4a878faf728c8e24a8fd_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,295 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117074e0c19805345a9b7362354f25943d4_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117074e0c19805345a9b7362354f25943d4_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,297 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117082292393df64486a5b642efc47e1ecc_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117082292393df64486a5b642efc47e1ecc_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,298 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117106747187c9d457b9b89cdfda36b5bd2_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117106747187c9d457b9b89cdfda36b5bd2_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,299 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117112a63072e084eaf9932f50224c70fd9_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117112a63072e084eaf9932f50224c70fd9_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,301 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111715dfee6f5dbb48cf9a9c2a54133b1b6f_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111715dfee6f5dbb48cf9a9c2a54133b1b6f_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,303 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172cbaec6337c14e2e9215def8e9a82e11_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172cbaec6337c14e2e9215def8e9a82e11_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,304 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730a452d65180480fa44ab8eee58e6ec2_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730a452d65180480fa44ab8eee58e6ec2_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,305 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175818b906ae2c4a7fbb6144ebafaffda2_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175818b906ae2c4a7fbb6144ebafaffda2_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,307 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177e26b26c02e1440db7c420f5e923436e_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177e26b26c02e1440db7c420f5e923436e_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,309 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e7b412c0e344bc69b572b2369c5ef13_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e7b412c0e344bc69b572b2369c5ef13_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,310 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111796a12ac29229455ab6f06bad8c0e074e_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111796a12ac29229455ab6f06bad8c0e074e_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,311 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a89c3b25258b4fcfb3cd64ff7df516a4_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a89c3b25258b4fcfb3cd64ff7df516a4_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,313 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c249271e1c57449ab5a5bf11795d028d_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c249271e1c57449ab5a5bf11795d028d_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,315 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dec09002d1874e2e91a3e9ee169671a3_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dec09002d1874e2e91a3e9ee169671a3_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,316 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ea1a3d5146764d428299c0a14e9df0c9_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ea1a3d5146764d428299c0a14e9df0c9_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,318 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f7170d4625434ca68655280334ff8ecd_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f7170d4625434ca68655280334ff8ecd_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,319 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fbdd8966381a4d899b52d0138de56061_ee6d6c86df174d97a57ee15dc309739b to hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fbdd8966381a4d899b52d0138de56061_ee6d6c86df174d97a57ee15dc309739b 2024-11-17T20:26:34,320 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T20:26:34,322 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=180, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:34,325 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T20:26:34,327 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T20:26:34,328 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=180, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:34,328 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T20:26:34,329 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731875194328"}]},"ts":"9223372036854775807"} 2024-11-17T20:26:34,331 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T20:26:34,331 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee6d6c86df174d97a57ee15dc309739b, NAME => 'TestAcidGuarantees,,1731875164381.ee6d6c86df174d97a57ee15dc309739b.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T20:26:34,331 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T20:26:34,331 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731875194331"}]},"ts":"9223372036854775807"} 2024-11-17T20:26:34,333 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T20:26:34,334 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=180, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T20:26:34,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-17T20:26:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-17T20:26:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T20:26:34,360 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T20:26:34,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 243), OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=482 (was 520), ProcessCount=11 (was 11), AvailableMemoryMB=7871 (was 8096) 2024-11-17T20:26:34,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T20:26:34,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:26:34,374 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:26:34,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:34,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:34,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T20:26:34,374 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T20:26:34,374 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1866486918, stopped=false 2024-11-17T20:26:34,375 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b0b0ae8de03,32801,1731875028288 2024-11-17T20:26:34,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T20:26:34,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T20:26:34,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:26:34,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:26:34,451 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T20:26:34,451 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T20:26:34,451 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:26:34,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:34,452 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b0b0ae8de03,46555,1731875029029' ***** 2024-11-17T20:26:34,452 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T20:26:34,452 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T20:26:34,452 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T20:26:34,452 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T20:26:34,453 INFO [RS:0;0b0b0ae8de03:46555 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T20:26:34,453 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T20:26:34,453 INFO [RS:0;0b0b0ae8de03:46555 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T20:26:34,453 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(959): stopping server 0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:34,453 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T20:26:34,453 INFO [RS:0;0b0b0ae8de03:46555 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b0b0ae8de03:46555. 2024-11-17T20:26:34,454 DEBUG [RS:0;0b0b0ae8de03:46555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T20:26:34,454 DEBUG [RS:0;0b0b0ae8de03:46555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:34,454 INFO [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T20:26:34,454 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T20:26:34,454 INFO [regionserver/0b0b0ae8de03:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T20:26:34,454 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T20:26:34,454 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T20:26:34,454 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T20:26:34,455 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T20:26:34,455 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T20:26:34,455 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T20:26:34,455 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:34,455 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T20:26:34,455 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T20:26:34,455 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T20:26:34,455 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T20:26:34,455 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T20:26:34,477 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/info/0ea518d7b79b41fb8216d1deb34f56e4 is 95, key is TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811./info:/1731875058355/DeleteFamily/seqid=0 2024-11-17T20:26:34,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742502_1678 (size=6514) 2024-11-17T20:26:34,526 INFO [regionserver/0b0b0ae8de03:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T20:26:34,656 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:34,856 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:34,881 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/info/0ea518d7b79b41fb8216d1deb34f56e4 2024-11-17T20:26:34,909 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/ns/5fb08d01ed9d404d92f62b6e158750f3 is 93, key is TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811./ns:/1731875058318/DeleteFamily/seqid=0 2024-11-17T20:26:34,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742503_1679 (size=5906) 2024-11-17T20:26:35,056 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:35,256 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:35,314 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/ns/5fb08d01ed9d404d92f62b6e158750f3 2024-11-17T20:26:35,340 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/rep_barrier/149a663e2c38461aa7d9588d4a1cbb07 is 102, key is TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811./rep_barrier:/1731875058318/DeleteFamily/seqid=0 2024-11-17T20:26:35,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742504_1680 (size=6025) 2024-11-17T20:26:35,457 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:35,657 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:35,744 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/rep_barrier/149a663e2c38461aa7d9588d4a1cbb07 2024-11-17T20:26:35,772 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/table/3df19dc6621245cba8e69fe9cf28910e is 96, key is TestAcidGuarantees,,1731875033041.ff04589cbd7acbe489756dc73d4cf811./table:/1731875058318/DeleteFamily/seqid=0 2024-11-17T20:26:35,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742505_1681 (size=5936) 2024-11-17T20:26:35,857 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:36,058 DEBUG [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T20:26:36,176 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/table/3df19dc6621245cba8e69fe9cf28910e 2024-11-17T20:26:36,186 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/info/0ea518d7b79b41fb8216d1deb34f56e4 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/info/0ea518d7b79b41fb8216d1deb34f56e4 2024-11-17T20:26:36,192 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/info/0ea518d7b79b41fb8216d1deb34f56e4, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T20:26:36,193 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/ns/5fb08d01ed9d404d92f62b6e158750f3 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/ns/5fb08d01ed9d404d92f62b6e158750f3 2024-11-17T20:26:36,198 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/ns/5fb08d01ed9d404d92f62b6e158750f3, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T20:26:36,198 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/rep_barrier/149a663e2c38461aa7d9588d4a1cbb07 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/rep_barrier/149a663e2c38461aa7d9588d4a1cbb07 2024-11-17T20:26:36,203 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/rep_barrier/149a663e2c38461aa7d9588d4a1cbb07, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T20:26:36,203 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/.tmp/table/3df19dc6621245cba8e69fe9cf28910e as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/table/3df19dc6621245cba8e69fe9cf28910e 2024-11-17T20:26:36,208 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/table/3df19dc6621245cba8e69fe9cf28910e, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T20:26:36,208 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1753ms, sequenceid=90, compaction requested=false 2024-11-17T20:26:36,213 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T20:26:36,214 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T20:26:36,214 INFO [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T20:26:36,214 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731875194455Running coprocessor pre-close hooks at 1731875194455Disabling compacts and flushes for region at 1731875194455Disabling writes for close at 1731875194455Obtaining lock to block concurrent updates at 1731875194456 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731875194456Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731875194456Flushing stores of hbase:meta,,1.1588230740 at 1731875194456Flushing 1588230740/info: creating writer at 1731875194457 (+1 ms)Flushing 1588230740/info: appending metadata at 1731875194477 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731875194477Flushing 1588230740/ns: creating writer at 1731875194892 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731875194909 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731875194909Flushing 1588230740/rep_barrier: creating writer at 1731875195322 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731875195340 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731875195340Flushing 1588230740/table: creating writer at 1731875195752 (+412 ms)Flushing 1588230740/table: appending metadata at 1731875195771 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731875195771Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eacc0f3: reopening flushed file at 1731875196185 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11b1879d: reopening flushed file at 1731875196192 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62c7e25: reopening flushed file at 1731875196198 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2afd586f: reopening flushed file at 1731875196203 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1753ms, sequenceid=90, compaction requested=false at 1731875196208 (+5 ms)Writing region close event to WAL at 1731875196210 (+2 ms)Running coprocessor post-close hooks at 1731875196214 (+4 ms)Closed at 1731875196214 2024-11-17T20:26:36,214 DEBUG [RS_CLOSE_META-regionserver/0b0b0ae8de03:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T20:26:36,258 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(976): stopping server 0b0b0ae8de03,46555,1731875029029; all regions closed. 2024-11-17T20:26:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741834_1010 (size=25344) 2024-11-17T20:26:36,270 DEBUG [RS:0;0b0b0ae8de03:46555 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/oldWALs 2024-11-17T20:26:36,270 INFO [RS:0;0b0b0ae8de03:46555 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b0b0ae8de03%2C46555%2C1731875029029.meta:.meta(num 1731875032594) 2024-11-17T20:26:36,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741832_1008 (size=15500817) 2024-11-17T20:26:36,273 DEBUG [RS:0;0b0b0ae8de03:46555 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/oldWALs 2024-11-17T20:26:36,273 INFO [RS:0;0b0b0ae8de03:46555 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b0b0ae8de03%2C46555%2C1731875029029:(num 1731875031622) 2024-11-17T20:26:36,273 DEBUG [RS:0;0b0b0ae8de03:46555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T20:26:36,273 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T20:26:36,273 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T20:26:36,273 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.ChoreService(370): Chore service for: regionserver/0b0b0ae8de03:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T20:26:36,273 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T20:26:36,273 INFO [regionserver/0b0b0ae8de03:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T20:26:36,274 INFO [RS:0;0b0b0ae8de03:46555 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46555 2024-11-17T20:26:36,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b0b0ae8de03,46555,1731875029029 2024-11-17T20:26:36,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T20:26:36,333 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T20:26:36,346 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b0b0ae8de03,46555,1731875029029] 2024-11-17T20:26:36,356 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b0b0ae8de03,46555,1731875029029 already deleted, retry=false 2024-11-17T20:26:36,356 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b0b0ae8de03,46555,1731875029029 expired; onlineServers=0 2024-11-17T20:26:36,356 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b0b0ae8de03,32801,1731875028288' ***** 2024-11-17T20:26:36,356 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T20:26:36,357 INFO [M:0;0b0b0ae8de03:32801 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T20:26:36,357 INFO [M:0;0b0b0ae8de03:32801 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T20:26:36,357 DEBUG [M:0;0b0b0ae8de03:32801 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T20:26:36,357 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T20:26:36,357 DEBUG [M:0;0b0b0ae8de03:32801 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T20:26:36,357 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875031272 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.large.0-1731875031272,5,FailOnTimeoutGroup] 2024-11-17T20:26:36,357 DEBUG [master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875031273 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b0b0ae8de03:0:becomeActiveMaster-HFileCleaner.small.0-1731875031273,5,FailOnTimeoutGroup] 2024-11-17T20:26:36,357 INFO [M:0;0b0b0ae8de03:32801 {}] hbase.ChoreService(370): Chore service for: master/0b0b0ae8de03:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T20:26:36,357 INFO [M:0;0b0b0ae8de03:32801 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T20:26:36,358 DEBUG [M:0;0b0b0ae8de03:32801 {}] master.HMaster(1795): Stopping service threads 2024-11-17T20:26:36,358 INFO [M:0;0b0b0ae8de03:32801 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T20:26:36,358 INFO [M:0;0b0b0ae8de03:32801 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T20:26:36,358 ERROR [M:0;0b0b0ae8de03:32801 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:42493 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:42493,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T20:26:36,359 INFO [M:0;0b0b0ae8de03:32801 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T20:26:36,359 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T20:26:36,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T20:26:36,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T20:26:36,367 DEBUG [M:0;0b0b0ae8de03:32801 {}] zookeeper.ZKUtil(347): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T20:26:36,367 WARN [M:0;0b0b0ae8de03:32801 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T20:26:36,367 INFO [M:0;0b0b0ae8de03:32801 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/.lastflushedseqids 2024-11-17T20:26:36,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742506_1682 (size=72) 2024-11-17T20:26:36,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T20:26:36,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46555-0x1014a77b5a50001, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T20:26:36,447 INFO [RS:0;0b0b0ae8de03:46555 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T20:26:36,448 INFO [RS:0;0b0b0ae8de03:46555 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b0b0ae8de03,46555,1731875029029; zookeeper connection closed. 2024-11-17T20:26:36,448 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a9d691a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a9d691a 2024-11-17T20:26:36,449 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T20:26:36,780 INFO [M:0;0b0b0ae8de03:32801 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T20:26:36,781 INFO [M:0;0b0b0ae8de03:32801 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T20:26:36,781 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T20:26:36,781 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:26:36,781 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:26:36,781 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T20:26:36,781 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:26:36,781 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=784.14 KB heapSize=965.59 KB 2024-11-17T20:26:36,802 DEBUG [M:0;0b0b0ae8de03:32801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc0423f678f94b798c2ec2565dad799b is 82, key is hbase:meta,,1/info:regioninfo/1731875032680/Put/seqid=0 2024-11-17T20:26:36,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742507_1683 (size=5672) 2024-11-17T20:26:37,206 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc0423f678f94b798c2ec2565dad799b 2024-11-17T20:26:37,225 DEBUG [M:0;0b0b0ae8de03:32801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08286ceb7a75446687870814d5b13d83 is 2276, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1731875061544/Put/seqid=0 2024-11-17T20:26:37,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742508_1684 (size=45940) 2024-11-17T20:26:37,629 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=783.59 KB at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08286ceb7a75446687870814d5b13d83 2024-11-17T20:26:37,632 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 08286ceb7a75446687870814d5b13d83 2024-11-17T20:26:37,646 DEBUG [M:0;0b0b0ae8de03:32801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0013e5641ff546cc88f427f4f65a1016 is 69, key is 0b0b0ae8de03,46555,1731875029029/rs:state/1731875031366/Put/seqid=0 2024-11-17T20:26:37,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073742509_1685 (size=5156) 2024-11-17T20:26:38,049 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0013e5641ff546cc88f427f4f65a1016 2024-11-17T20:26:38,052 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc0423f678f94b798c2ec2565dad799b as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc0423f678f94b798c2ec2565dad799b 2024-11-17T20:26:38,054 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc0423f678f94b798c2ec2565dad799b, entries=8, sequenceid=2241, filesize=5.5 K 2024-11-17T20:26:38,055 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08286ceb7a75446687870814d5b13d83 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/08286ceb7a75446687870814d5b13d83 2024-11-17T20:26:38,057 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 08286ceb7a75446687870814d5b13d83 2024-11-17T20:26:38,057 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/08286ceb7a75446687870814d5b13d83, entries=180, sequenceid=2241, filesize=44.9 K 2024-11-17T20:26:38,058 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0013e5641ff546cc88f427f4f65a1016 as hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0013e5641ff546cc88f427f4f65a1016 2024-11-17T20:26:38,060 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42493/user/jenkins/test-data/3e1a6da4-f183-23e1-7ef1-b7db339aa828/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0013e5641ff546cc88f427f4f65a1016, entries=1, sequenceid=2241, filesize=5.0 K 2024-11-17T20:26:38,061 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(3140): Finished flush of dataSize ~784.14 KB/802964, heapSize ~965.30 KB/988464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1280ms, sequenceid=2241, compaction requested=false 2024-11-17T20:26:38,062 INFO [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T20:26:38,062 DEBUG [M:0;0b0b0ae8de03:32801 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731875196781Disabling compacts and flushes for region at 1731875196781Disabling writes for close at 1731875196781Obtaining lock to block concurrent updates at 1731875196781Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731875196781Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=802964, getHeapSize=988704, getOffHeapSize=0, getCellsCount=2446 at 1731875196781Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731875196782 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731875196782Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731875196802 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731875196802Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731875197209 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731875197225 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731875197225Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731875197632 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731875197645 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731875197645Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@273cacd1: reopening flushed file at 1731875198052 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ba3ccd5: reopening flushed file at 1731875198055 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@233f8771: reopening flushed file at 1731875198057 (+2 ms)Finished flush of dataSize ~784.14 KB/802964, heapSize ~965.30 KB/988464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1280ms, sequenceid=2241, compaction requested=false at 1731875198061 (+4 ms)Writing region close event to WAL at 1731875198062 (+1 ms)Closed at 1731875198062 2024-11-17T20:26:38,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38093 is added to blk_1073741830_1006 (size=949868) 2024-11-17T20:26:38,065 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T20:26:38,065 INFO [M:0;0b0b0ae8de03:32801 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T20:26:38,065 INFO [M:0;0b0b0ae8de03:32801 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32801 2024-11-17T20:26:38,065 INFO [M:0;0b0b0ae8de03:32801 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T20:26:38,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T20:26:38,182 INFO [M:0;0b0b0ae8de03:32801 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T20:26:38,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32801-0x1014a77b5a50000, quorum=127.0.0.1:60122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T20:26:38,186 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T20:26:38,188 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T20:26:38,189 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T20:26:38,189 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T20:26:38,189 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/hadoop.log.dir/,STOPPED} 2024-11-17T20:26:38,191 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T20:26:38,191 WARN [BP-1443374901-172.17.0.2-1731875024098 heartbeating to localhost/127.0.0.1:42493 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T20:26:38,191 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T20:26:38,191 WARN [BP-1443374901-172.17.0.2-1731875024098 heartbeating to localhost/127.0.0.1:42493 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1443374901-172.17.0.2-1731875024098 (Datanode Uuid 1404d3bf-c966-4ccb-85d3-420dc03811ed) service to localhost/127.0.0.1:42493 2024-11-17T20:26:38,194 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/data/data1/current/BP-1443374901-172.17.0.2-1731875024098 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T20:26:38,195 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/cluster_63b22139-7e95-2d9f-25d6-0f7b4549b6ac/data/data2/current/BP-1443374901-172.17.0.2-1731875024098 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T20:26:38,195 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T20:26:38,203 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T20:26:38,204 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T20:26:38,204 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T20:26:38,204 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T20:26:38,204 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4472c32-be02-63d1-2454-5e8306b4aff5/hadoop.log.dir/,STOPPED} 2024-11-17T20:26:38,218 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T20:26:38,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down